From 7274e4b7532bacf94da255669f69dc851331e6ce Mon Sep 17 00:00:00 2001 From: arnavb Date: Sun, 23 Nov 2025 14:05:27 +0000 Subject: [PATCH 01/24] Add FSST support --- cpp/src/parquet/CMakeLists.txt | 2 + cpp/src/parquet/column_reader.cc | 3 +- cpp/src/parquet/decoder.cc | 262 +++++++ cpp/src/parquet/encoder.cc | 180 ++++- cpp/src/parquet/encoding_test.cc | 310 +++++++++ cpp/src/parquet/parquet.thrift | 7 + cpp/src/parquet/thirdparty/fsst/fsst.cpp | 200 ++++++ cpp/src/parquet/thirdparty/fsst/fsst.h | 227 ++++++ .../parquet/thirdparty/fsst/fsst_avx512.cpp | 149 ++++ .../parquet/thirdparty/fsst/fsst_avx512.inc | 57 ++ .../thirdparty/fsst/fsst_avx512_unroll1.inc | 57 ++ .../thirdparty/fsst/fsst_avx512_unroll2.inc | 114 +++ .../thirdparty/fsst/fsst_avx512_unroll3.inc | 171 +++++ .../thirdparty/fsst/fsst_avx512_unroll4.inc | 228 ++++++ cpp/src/parquet/thirdparty/fsst/libfsst.cpp | 651 ++++++++++++++++++ cpp/src/parquet/thirdparty/fsst/libfsst.hpp | 471 +++++++++++++ cpp/src/parquet/types.cc | 2 + cpp/src/parquet/types.h | 3 +- python/pyarrow/_parquet.pyx | 2 + python/pyarrow/includes/libparquet.pxd | 1 + python/pyarrow/parquet/core.py | 2 +- 21 files changed, 3095 insertions(+), 4 deletions(-) create mode 100644 cpp/src/parquet/thirdparty/fsst/fsst.cpp create mode 100644 cpp/src/parquet/thirdparty/fsst/fsst.h create mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512.cpp create mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512.inc create mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll1.inc create mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll2.inc create mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll3.inc create mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll4.inc create mode 100644 cpp/src/parquet/thirdparty/fsst/libfsst.cpp create mode 100644 cpp/src/parquet/thirdparty/fsst/libfsst.hpp diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index dc7d40d2a386..e06d13d1884e 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -171,6 +171,8 @@ set(PARQUET_SRCS encryption/internal_file_encryptor.cc exception.cc file_reader.cc + thirdparty/fsst/libfsst.cpp + thirdparty/fsst/fsst_avx512.cpp file_writer.cc geospatial/statistics.cc geospatial/util_internal.cc diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc index 8ecb774022f0..59c3b9934a24 100644 --- a/cpp/src/parquet/column_reader.cc +++ b/cpp/src/parquet/column_reader.cc @@ -862,7 +862,8 @@ class ColumnReaderImplBase { case Encoding::RLE: case Encoding::DELTA_BINARY_PACKED: case Encoding::DELTA_BYTE_ARRAY: - case Encoding::DELTA_LENGTH_BYTE_ARRAY: { + case Encoding::DELTA_LENGTH_BYTE_ARRAY: + case Encoding::FSST: { auto decoder = MakeTypedDecoder(encoding, descr_, pool_); current_decoder_ = decoder.get(); decoders_[static_cast(encoding)] = std::move(decoder); diff --git a/cpp/src/parquet/decoder.cc b/cpp/src/parquet/decoder.cc index d0a857dd22ac..77bc3c760ff2 100644 --- a/cpp/src/parquet/decoder.cc +++ b/cpp/src/parquet/decoder.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include "arrow/array.h" #include "arrow/array/builder_binary.h" @@ -51,6 +53,7 @@ #include "parquet/exception.h" #include "parquet/platform.h" #include "parquet/schema.h" +#include "parquet/thirdparty/fsst/fsst.h" #include "parquet/types.h" #ifdef _MSC_VER @@ -2307,6 +2310,258 @@ class ByteStreamSplitDecoder : public ByteStreamSplitDecoderBase { + public: + using Base = DecoderImpl; + using Base::num_values_; + + explicit FsstDecoder(const ColumnDescriptor* descr, + ::arrow::MemoryPool* pool = ::arrow::default_memory_pool()) + : DecoderImpl(descr, Encoding::FSST), pool_(pool) {} + + void SetData(int num_values, const uint8_t* data, int len) override { + const auto header_size = static_cast(sizeof(fsst_decoder_t)); + if (len < header_size) { + throw ParquetException("FSST page too small to contain decoder header"); + } + num_values_ = num_values; + memcpy(&decoder_, data, sizeof(fsst_decoder_t)); + next_ = data + header_size; + remaining_bytes_ = len - header_size; + decode_buffer_size_ = 0; + } + + int Decode(ByteArray* buffer, int max_values) override { + max_values = std::min(max_values, num_values_); + if (max_values == 0) { + return 0; + } + + const int64_t estimated_output_size = + decode_buffer_size_ + + static_cast(remaining_bytes_) * kMaxDecompressionExpansion; + EnsureDecodeBuffer(estimated_output_size); + + int decoded = 0; + + while (decoded < max_values) { + if (ARROW_PREDICT_FALSE(remaining_bytes_ < static_cast(kLengthPrefixSize))) { + throw ParquetException("FSST data truncated before length prefix"); + } + + const uint32_t compressed_len = SafeLoadAs(next_); + next_ += kLengthPrefixSize; + remaining_bytes_ -= static_cast(kLengthPrefixSize); + + if (ARROW_PREDICT_FALSE(compressed_len > static_cast(remaining_bytes_))) { + throw ParquetException("FSST compressed length exceeds available data"); + } + + const uint8_t* compressed_ptr = next_; + next_ += compressed_len; + remaining_bytes_ -= static_cast(compressed_len); + + uint8_t* value_ptr = nullptr; + const size_t decompressed_len = + DecompressValue(compressed_ptr, compressed_len, &value_ptr); + + buffer[decoded].ptr = value_ptr; + buffer[decoded].len = static_cast(decompressed_len); + decode_buffer_size_ += static_cast(decompressed_len); + ++decoded; + } + + num_values_ -= decoded; + return decoded; + } + + int DecodeSpaced(ByteArray* buffer, int num_values, int null_count, + const uint8_t* valid_bits, + int64_t valid_bits_offset) override { + if (null_count == 0) { + return Decode(buffer, num_values); + } + + const int values_to_decode = num_values - null_count; + temp_values_.resize(values_to_decode); + const int decoded = Decode(temp_values_.data(), values_to_decode); + if (ARROW_PREDICT_FALSE(decoded != values_to_decode)) { + throw ParquetException("Expected to decode ", values_to_decode, + " values but decoded ", decoded, " values."); + } + + int value_index = 0; + for (int i = 0; i < num_values; ++i) { + if (bit_util::GetBit(valid_bits, valid_bits_offset + i)) { + buffer[i] = temp_values_[value_index++]; + } else { + buffer[i].ptr = nullptr; + buffer[i].len = 0; + } + } + + return value_index; + } + + int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::Accumulator* builder) override { + int values_decoded = 0; + PARQUET_THROW_NOT_OK( + DecodeArrowDense(num_values, null_count, valid_bits, valid_bits_offset, builder, + &values_decoded)); + return values_decoded; + } + + int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::DictAccumulator* builder) override { + int values_decoded = 0; + PARQUET_THROW_NOT_OK( + DecodeArrowDict(num_values, null_count, valid_bits, valid_bits_offset, builder, + &values_decoded)); + return values_decoded; + } + + private: + Status DecodeArrowDense(int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::Accumulator* out, + int* out_values_decoded) { + const int values_to_decode = num_values - null_count; + temp_values_.resize(values_to_decode); + + const int decoded = Decode(temp_values_.data(), values_to_decode); + if (ARROW_PREDICT_FALSE(decoded != values_to_decode)) { + throw ParquetException("Expected to decode ", values_to_decode, + " values but decoded ", decoded, " values."); + } + + auto visit_binary_helper = [&](auto* helper) { + auto* values_ptr = temp_values_.data(); + int value_index = 0; + + RETURN_NOT_OK( + VisitBitRuns(valid_bits, valid_bits_offset, num_values, + [&](int64_t position, int64_t run_length, bool is_valid) { + if (is_valid) { + for (int64_t i = 0; i < run_length; ++i) { + const auto& value = values_ptr[value_index++]; + RETURN_NOT_OK(helper->AppendValue( + value.ptr, static_cast(value.len))); + } + } else { + RETURN_NOT_OK(helper->AppendNulls(run_length)); + } + return Status::OK(); + })); + + *out_values_decoded = decoded; + return Status::OK(); + }; + + return DispatchArrowBinaryHelper( + out, num_values, /*estimated_data_length=*/{}, visit_binary_helper); + } + + Status DecodeArrowDict(int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::DictAccumulator* builder, + int* out_values_decoded) { + const int values_to_decode = num_values - null_count; + temp_values_.resize(values_to_decode); + + const int decoded = Decode(temp_values_.data(), values_to_decode); + if (ARROW_PREDICT_FALSE(decoded != values_to_decode)) { + throw ParquetException("Expected to decode ", values_to_decode, + " values but decoded ", decoded, " values."); + } + + RETURN_NOT_OK(builder->Reserve(num_values)); + + int value_index = 0; + RETURN_NOT_OK(VisitBitRuns( + valid_bits, valid_bits_offset, num_values, + [&](int64_t position, int64_t run_length, bool is_valid) { + if (is_valid) { + for (int64_t i = 0; i < run_length; ++i) { + const auto& value = temp_values_[value_index++]; + RETURN_NOT_OK(builder->Append(value.ptr, static_cast(value.len))); + } + } else { + RETURN_NOT_OK(builder->AppendNulls(run_length)); + } + return Status::OK(); + })); + + *out_values_decoded = decoded; + return Status::OK(); + } + + uint8_t* EnsureDecodeBuffer(int64_t capacity) { + const int64_t min_capacity = + std::max(capacity, kInitialDecodeBufferSize); + const int64_t target = ::arrow::bit_util::NextPower2(min_capacity); + + if (!decode_buffer_) { + PARQUET_ASSIGN_OR_THROW( + decode_buffer_, ::arrow::AllocateResizableBuffer(target, pool_)); + } else if (decode_buffer_->size() < target) { + PARQUET_THROW_NOT_OK(decode_buffer_->Resize(target, false)); + } + return decode_buffer_->mutable_data(); + } + + size_t DecompressValue(const uint8_t* compressed_ptr, uint32_t compressed_len, + uint8_t** value_ptr) { + EnsureDecodeBuffer(decode_buffer_size_ + + OutputUpperBound(compressed_len)); + + while (true) { + uint8_t* destination = decode_buffer_->mutable_data() + decode_buffer_size_; + const size_t available = + static_cast(decode_buffer_->size() - decode_buffer_size_); + + const size_t decompressed = + fsst_decompress(&decoder_, compressed_len, compressed_ptr, available, + destination); + + if (decompressed > 0 || compressed_len == 0) { + *value_ptr = destination; + return decompressed; + } + + int64_t new_capacity = std::max( + decode_buffer_->size() * 2, + decode_buffer_size_ + OutputUpperBound(compressed_len)); + if (new_capacity <= decode_buffer_->size()) { + throw ParquetException("FSST decompression failed"); + } + EnsureDecodeBuffer(new_capacity); + } + } + + static int64_t OutputUpperBound(uint32_t compressed_len) { + const int64_t expanded = + static_cast(compressed_len) * kMaxDecompressionExpansion; + return std::max(expanded, kInitialDecodeBufferSize); + } + + static constexpr size_t kLengthPrefixSize = sizeof(uint32_t); + static constexpr int64_t kInitialDecodeBufferSize = 1024; + static constexpr int64_t kMaxDecompressionExpansion = 8; + + fsst_decoder_t decoder_{}; + const uint8_t* next_ = nullptr; + int remaining_bytes_ = 0; + ::arrow::MemoryPool* pool_; + std::shared_ptr<::arrow::ResizableBuffer> decode_buffer_; + int64_t decode_buffer_size_ = 0; + std::vector temp_values_; +}; + } // namespace // ---------------------------------------------------------------------- @@ -2373,6 +2628,13 @@ std::unique_ptr MakeDecoder(Type::type type_num, Encoding::type encodin throw ParquetException( "DELTA_BYTE_ARRAY only supports BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY"); } + } else if (encoding == Encoding::FSST) { + switch (type_num) { + case Type::BYTE_ARRAY: + return std::make_unique(descr, pool); + default: + throw ParquetException("FSST encoding only supports BYTE_ARRAY"); + } } else if (encoding == Encoding::DELTA_LENGTH_BYTE_ARRAY) { if (type_num == Type::BYTE_ARRAY) { return std::make_unique(descr, pool); diff --git a/cpp/src/parquet/encoder.cc b/cpp/src/parquet/encoder.cc index f9367555d97f..61b1bb9606c3 100644 --- a/cpp/src/parquet/encoder.cc +++ b/cpp/src/parquet/encoder.cc @@ -18,8 +18,11 @@ #include "parquet/encoding.h" #include +#include #include #include +#include +#include #include #include #include @@ -47,6 +50,7 @@ #include "parquet/exception.h" #include "parquet/platform.h" #include "parquet/schema.h" +#include "parquet/thirdparty/fsst/fsst.h" #include "parquet/types.h" #ifdef _MSC_VER @@ -101,7 +105,7 @@ class EncoderImpl : virtual public Encoder { MemoryPool* memory_pool() const override { return pool_; } int64_t ReportUnencodedDataBytes() override { - if (descr_->physical_type() != Type::BYTE_ARRAY) { + if (descr_ != nullptr && descr_->physical_type() != Type::BYTE_ARRAY) { throw ParquetException("ReportUnencodedDataBytes is only supported for BYTE_ARRAY"); } int64_t bytes = unencoded_byte_array_data_bytes_; @@ -1737,6 +1741,173 @@ std::shared_ptr RleBooleanEncoder::FlushValues() { return buffer; } +// ---------------------------------------------------------------------- + +class FsstEncoder : public EncoderImpl, virtual public TypedEncoder { + public: + explicit FsstEncoder(const ColumnDescriptor* descr, MemoryPool* pool) + : EncoderImpl(descr, Encoding::FSST, pool), + encoder_(nullptr), + unencoded_values_() {} + + ~FsstEncoder() override { + if (encoder_) { + fsst_destroy(encoder_); + } + } + + int64_t EstimatedDataEncodedSize() override { + const int64_t total_size = pending_unencoded_bytes_; + const double scaled = + static_cast(total_size) * compression_ratio_hint_; + const int64_t estimated_payload = + static_cast(std::ceil(scaled)); + return static_cast(sizeof(fsst_decoder_t)) + + std::max(0, estimated_payload); + } + + std::shared_ptr FlushValues() override { + if (unencoded_values_.empty()) { + PARQUET_ASSIGN_OR_THROW(auto buffer, AllocateBuffer(0, pool_)); + return buffer; + } + + BuildSymbolTable(); + + const int64_t total_input_size = pending_unencoded_bytes_; + + const int64_t decoder_bytes = static_cast(sizeof(fsst_decoder_t)); + const int64_t length_prefix_bytes = + static_cast(unencoded_values_.size()) * static_cast(sizeof(uint32_t)); + const int64_t estimated_buffer_size = + decoder_bytes + total_input_size * kFsstCompressionExpansion + length_prefix_bytes; + + PARQUET_ASSIGN_OR_THROW(auto output_buffer, + AllocateResizableBuffer(estimated_buffer_size, pool_)); + uint8_t* out_ptr = output_buffer->mutable_data(); + + fsst_decoder_t decoder = fsst_decoder(encoder_); + memcpy(out_ptr, &decoder, sizeof(fsst_decoder_t)); + out_ptr += sizeof(fsst_decoder_t); + + int64_t total_output_size = sizeof(fsst_decoder_t); + std::vector out_lengths(1); + std::vector out_ptrs(1); + + for (size_t i = 0; i < unencoded_values_.size(); ++i) { + const int64_t remaining_capacity = + estimated_buffer_size - total_output_size - sizeof(uint32_t); + if (ARROW_PREDICT_FALSE(remaining_capacity <= 0)) { + throw ParquetException("FSST compression buffer exhausted"); + } + size_t available = static_cast(remaining_capacity); + size_t input_length = static_cast(unencoded_values_[i].len); + const unsigned char* input_ptr = unencoded_values_[i].ptr; + size_t num_compressed = + fsst_compress(encoder_, 1, &input_length, &input_ptr, available, + out_ptr + sizeof(uint32_t), out_lengths.data(), out_ptrs.data()); + + if (num_compressed == 0) { + throw ParquetException("FSST compression buffer too small"); + } + + uint32_t len = static_cast(out_lengths[0]); + memcpy(out_ptr, &len, sizeof(uint32_t)); + out_ptr += sizeof(uint32_t) + out_lengths[0]; + total_output_size += sizeof(uint32_t) + out_lengths[0]; + } + + PARQUET_THROW_NOT_OK(output_buffer->Resize(total_output_size)); + UpdateCompressionStats(total_input_size, total_output_size); + unencoded_values_.clear(); + pending_unencoded_bytes_ = 0; + unencoded_byte_array_data_bytes_ = 0; + return output_buffer; + } + + using TypedEncoder::Put; + + void Put(const ByteArray* src, int num_values) override { + for (int i = 0; i < num_values; ++i) { + unencoded_values_.push_back(src[i]); + const int64_t length = static_cast(src[i].len); + unencoded_byte_array_data_bytes_ += length; + pending_unencoded_bytes_ += length; + } + } + + void Put(const ::arrow::Array& values) override { + AssertVarLengthBinary(values); + + PARQUET_THROW_NOT_OK(::arrow::VisitArraySpanInline<::arrow::BinaryType>( + *values.data(), + [&](::std::string_view view) { + if (ARROW_PREDICT_FALSE(view.size() > kMaxByteArraySize)) { + return Status::Invalid("Parquet cannot store strings with size 2GB or more"); + } + ByteArray val; + val.len = static_cast(view.size()); + val.ptr = reinterpret_cast(view.data()); + unencoded_values_.push_back(val); + const int64_t length = static_cast(val.len); + unencoded_byte_array_data_bytes_ += length; + pending_unencoded_bytes_ += length; + return Status::OK(); + }, + []() { return Status::OK(); })); + } + + void PutSpaced(const ByteArray* src, int num_values, const uint8_t* valid_bits, + int64_t valid_bits_offset) override { + if (valid_bits != NULLPTR) { + PARQUET_ASSIGN_OR_THROW(auto buffer, ::arrow::AllocateBuffer(num_values * sizeof(ByteArray), pool_)); + auto buffer_ptr = reinterpret_cast(buffer->mutable_data()); + int num_valid_values = ::arrow::util::internal::SpacedCompress( + src, num_values, valid_bits, valid_bits_offset, buffer_ptr); + Put(buffer_ptr, num_valid_values); + } else { + Put(src, num_values); + } + } + + private: + void BuildSymbolTable() { + if (unencoded_values_.empty()) { + return; + } + + std::vector input_lengths; + std::vector input_ptrs; + + for (const auto& val : unencoded_values_) { + input_lengths.push_back(val.len); + input_ptrs.push_back(val.ptr); + } + + encoder_ = fsst_create(unencoded_values_.size(), input_lengths.data(), + input_ptrs.data(), 0); + + if (!encoder_) { + throw ParquetException("Failed to create FSST encoder"); + } + } + + void UpdateCompressionStats(int64_t input_bytes, int64_t payload_bytes) { + const double ratio = static_cast(std::max(0, payload_bytes)) / + static_cast(std::max(int64_t{1}, input_bytes)); + compression_ratio_hint_ = std::max(kMinimumCompressionRatio, ratio); + } + + static constexpr double kDefaultCompressionRatio = 1.0; + static constexpr double kMinimumCompressionRatio = 0.01; + // Allocate worst-case 4x expansion. + static constexpr int64_t kFsstCompressionExpansion = 4; + fsst_encoder_t* encoder_; + std::vector unencoded_values_; + double compression_ratio_hint_ = kDefaultCompressionRatio; + int64_t pending_unencoded_bytes_ = 0; +}; + } // namespace // ---------------------------------------------------------------------- @@ -1821,6 +1992,13 @@ std::unique_ptr MakeEncoder(Type::type type_num, Encoding::type encodin default: throw ParquetException("DELTA_LENGTH_BYTE_ARRAY only supports BYTE_ARRAY"); } + } else if (encoding == Encoding::FSST) { + switch (type_num) { + case Type::BYTE_ARRAY: + return std::make_unique(descr, pool); + default: + throw ParquetException("FSST encoding only supports BYTE_ARRAY"); + } } else if (encoding == Encoding::RLE) { switch (type_num) { case Type::BOOLEAN: diff --git a/cpp/src/parquet/encoding_test.cc b/cpp/src/parquet/encoding_test.cc index 66a3f7647fa8..ca51d8e1b241 100644 --- a/cpp/src/parquet/encoding_test.cc +++ b/cpp/src/parquet/encoding_test.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -28,7 +29,9 @@ #include "arrow/array/builder_binary.h" #include "arrow/array/builder_dict.h" #include "arrow/array/concatenate.h" +#include "arrow/buffer.h" #include "arrow/compute/cast.h" +#include "arrow/io/memory.h" #include "arrow/testing/gtest_util.h" #include "arrow/testing/random.h" #include "arrow/testing/util.h" @@ -41,10 +44,16 @@ #include "arrow/util/endian.h" #include "arrow/util/span.h" #include "arrow/util/string.h" +#include "parquet/column_page.h" +#include "parquet/column_reader.h" #include "parquet/encoding.h" +#include "parquet/file_reader.h" +#include "parquet/file_writer.h" +#include "parquet/exception.h" #include "parquet/platform.h" #include "parquet/schema.h" #include "parquet/test_util.h" +#include "parquet/thirdparty/fsst/fsst.h" #include "parquet/types.h" using arrow::default_memory_pool; @@ -2593,4 +2602,305 @@ TEST(DeltaByteArrayEncodingAdHoc, ArrowDirectPut) { } } +// ---------------------------------------------------------------------- + +TEST(TestFsstEncoding, BasicRoundTrip) { + ::arrow::random::RandomArrayGenerator rag(0); + constexpr int64_t kNumValues = 2048; + constexpr int64_t kNumUnique = 128; + constexpr int32_t kMinLength = 4; + constexpr int32_t kMaxLength = 48; + + auto values = rag.BinaryWithRepeats(kNumValues, kNumUnique, kMinLength, kMaxLength, + 0.0); + + auto encoder = MakeTypedEncoder(Encoding::FSST); + ASSERT_NO_THROW(encoder->Put(*values)); + auto encoded = encoder->FlushValues(); + ASSERT_NE(nullptr, encoded); + ASSERT_GT(encoded->size(), 0); + + auto decoder = MakeTypedDecoder(Encoding::FSST); + decoder->SetData(static_cast(values->length()), encoded->data(), + static_cast(encoded->size())); + + std::vector decoded(values->length()); + ASSERT_EQ(static_cast(values->length()), + decoder->Decode(decoded.data(), static_cast(decoded.size()))); + + const auto& binary = checked_cast(*values); + for (int64_t i = 0; i < values->length(); ++i) { + auto view = binary.GetView(i); + ASSERT_EQ(view.size(), static_cast(decoded[i].len)) << i; + ASSERT_EQ(0, memcmp(decoded[i].ptr, view.data(), view.size())) << i; + } +} + +TEST(TestFsstEncoding, FlushEmptyReturnsEmptyBuffer) { + auto encoder = MakeTypedEncoder(Encoding::FSST); + auto buffer = encoder->FlushValues(); + ASSERT_NE(nullptr, buffer); + ASSERT_EQ(0, buffer->size()); +} + +TEST(TestFsstEncoding, ReportUnencodedBytesResetsCounter) { + ::arrow::random::RandomArrayGenerator rag(42); + auto values = rag.String(256, 1, 24, 0.0); + + auto encoder = MakeTypedEncoder(Encoding::FSST); + ASSERT_NO_THROW(encoder->Put(*values)); + + const auto& binary = checked_cast(*values); + const int64_t expected = binary.total_values_length(); + EXPECT_EQ(expected, encoder->ReportUnencodedDataBytes()); + EXPECT_EQ(0, encoder->ReportUnencodedDataBytes()); + + auto encoded = encoder->FlushValues(); + ASSERT_NE(nullptr, encoded); + auto decoder = MakeTypedDecoder(Encoding::FSST); + decoder->SetData(static_cast(values->length()), encoded->data(), + static_cast(encoded->size())); + + std::vector decoded(values->length()); + ASSERT_EQ(static_cast(values->length()), + decoder->Decode(decoded.data(), static_cast(decoded.size()))); + + for (int64_t i = 0; i < values->length(); ++i) { + auto view = binary.GetView(i); + ASSERT_EQ(view.size(), static_cast(decoded[i].len)) << i; + ASSERT_EQ(0, memcmp(decoded[i].ptr, view.data(), view.size())) << i; + } + + EXPECT_EQ(0, encoder->ReportUnencodedDataBytes()); +} + +TEST(TestFsstEncoding, MultiPageRoundTrip) { + constexpr int64_t kStringsPerPage = 10000; + constexpr int64_t kNumPages = 2; + constexpr int64_t kTotalValues = kStringsPerPage * kNumPages; + constexpr int64_t kPageSizeBytes = 512 * 1024; + + std::vector page1; + std::vector page2; + page1.reserve(kStringsPerPage); + page2.reserve(kStringsPerPage); + + for (int64_t i = 0; i < kStringsPerPage; ++i) { + page1.push_back("https://www.example.com/user/profile/page" + std::to_string(i) + + "/settings/preferences/advanced/options"); + page2.push_back("ftp://ftp.downloads.org/pub/software/release" + std::to_string(i) + + "/package/installer/binaries/archive"); + } + + std::vector expected; + expected.reserve(kTotalValues); + expected.insert(expected.end(), page1.begin(), page1.end()); + expected.insert(expected.end(), page2.begin(), page2.end()); + + ASSERT_OK_AND_ASSIGN(auto output_stream, ::arrow::io::BufferOutputStream::Create()); + + schema::NodeVector fields; + fields.push_back(schema::PrimitiveNode::Make("url", Repetition::REQUIRED, + Type::BYTE_ARRAY, ConvertedType::UTF8)); + auto parquet_schema = std::static_pointer_cast( + schema::GroupNode::Make("schema", Repetition::REQUIRED, fields)); + + auto writer_props = WriterProperties::Builder() + .encoding(Encoding::FSST) + ->data_pagesize(kPageSizeBytes) + ->build(); + + std::unique_ptr writer; + ASSERT_NO_THROW(writer = + ParquetFileWriter::Open(output_stream, parquet_schema, writer_props)); + ASSERT_NE(nullptr, writer); + auto* row_group_writer = writer->AppendRowGroup(); + ASSERT_NE(nullptr, row_group_writer); + auto* column_writer = static_cast*>( + row_group_writer->NextColumn()); + ASSERT_NE(nullptr, column_writer); + + auto write_page = [&](const std::vector& source) { + std::vector batch; + batch.reserve(source.size()); + for (const auto& value : source) { + batch.emplace_back(value); + } + column_writer->WriteBatch(static_cast(batch.size()), nullptr, nullptr, + batch.data()); + }; + + write_page(page1); + write_page(page2); + + column_writer->Close(); + row_group_writer->Close(); + ASSERT_NO_THROW(writer->Close()); + + ASSERT_OK_AND_ASSIGN(auto buffer, output_stream->Finish()); + + auto make_reader = [&buffer]() -> std::unique_ptr { + auto buffer_reader = std::make_shared<::arrow::io::BufferReader>(buffer); + return ParquetFileReader::Open(buffer_reader); + }; + + std::unique_ptr page_reader_file; + ASSERT_NO_THROW(page_reader_file = make_reader()); + ASSERT_NE(nullptr, page_reader_file); + auto page_row_group = page_reader_file->RowGroup(0); + auto page_reader = page_row_group->GetColumnPageReader(0); + + int data_page_count = 0; + while (page_reader->NextPage() != nullptr) { + ++data_page_count; + } + EXPECT_GT(data_page_count, 1); + + std::unique_ptr reader; + ASSERT_NO_THROW(reader = make_reader()); + ASSERT_NE(nullptr, reader); + auto row_group_reader = reader->RowGroup(0); + auto column_reader = + std::static_pointer_cast>(row_group_reader->Column(0)); + + std::vector decoded(kTotalValues); + int64_t values_read = 0; + while (values_read < kTotalValues) { + int64_t batch_length = std::min(1024, kTotalValues - values_read); + int64_t batch_read = 0; + column_reader->ReadBatch(batch_length, nullptr, nullptr, + decoded.data() + values_read, &batch_read); + ASSERT_GT(batch_read, 0); + values_read += batch_read; + } + ASSERT_EQ(values_read, kTotalValues); + + for (int64_t i = 0; i < kTotalValues; ++i) { + ASSERT_EQ(expected[i].size(), static_cast(decoded[i].len)) << i; + ASSERT_EQ(0, memcmp(expected[i].data(), decoded[i].ptr, expected[i].size())) << i; + } +} + +TEST(TestFsstEncoding, MultipleFlushesProduceIndependentPages) { + ::arrow::random::RandomArrayGenerator rag(99); + auto encoder = MakeTypedEncoder(Encoding::FSST); + + const std::vector> batches = { + {64, 3, 12}, {512, 2, 24}, {17, 1, 8}}; + + for (const auto& [size, min_len, max_len] : batches) { + auto batch = rag.String(size, min_len, max_len, 0.0); + + ASSERT_NO_THROW(encoder->Put(*batch)); + auto buffer = encoder->FlushValues(); + ASSERT_NE(nullptr, buffer); + ASSERT_GT(buffer->size(), 0); + + auto decoder = MakeTypedDecoder(Encoding::FSST); + decoder->SetData(static_cast(batch->length()), buffer->data(), + static_cast(buffer->size())); + + std::vector decoded(batch->length()); + ASSERT_EQ(static_cast(batch->length()), + decoder->Decode(decoded.data(), static_cast(decoded.size()))); + + const auto& binary = checked_cast(*batch); + for (int64_t i = 0; i < batch->length(); ++i) { + auto view = binary.GetView(i); + ASSERT_EQ(view.size(), static_cast(decoded[i].len)) << i; + ASSERT_EQ(0, memcmp(decoded[i].ptr, view.data(), view.size())) << i; + } + + encoder->ReportUnencodedDataBytes(); + } +} + +TEST(TestFsstEncoding, DecodeRejectsOverstatedLength) { + ::arrow::random::RandomArrayGenerator rag(123); + auto values = rag.String(64, 3, 32, 0.0); + + auto encoder = MakeTypedEncoder(Encoding::FSST); + ASSERT_NO_THROW(encoder->Put(*values)); + auto encoded = encoder->FlushValues(); + ASSERT_NE(nullptr, encoded); + ASSERT_GT(encoded->size(), + static_cast(sizeof(fsst_decoder_t) + sizeof(uint32_t))); + + ASSERT_OK_AND_ASSIGN(auto corrupted, + ::arrow::AllocateBuffer(encoded->size(), default_memory_pool())); + std::memcpy(corrupted->mutable_data(), encoded->data(), encoded->size()); + + auto* length_ptr = + reinterpret_cast(corrupted->mutable_data() + sizeof(fsst_decoder_t)); + *length_ptr = static_cast(encoded->size()); + + auto decoder = MakeTypedDecoder(Encoding::FSST); + decoder->SetData(static_cast(values->length()), corrupted->data(), + static_cast(corrupted->size())); + + std::vector decoded(values->length()); + EXPECT_THROW(decoder->Decode(decoded.data(), static_cast(decoded.size())), + ParquetException); +} + +TEST(TestFsstEncoding, SetDataRejectsShortHeader) { + auto decoder = MakeTypedDecoder(Encoding::FSST); + + const int num_values = 1; + const int header_bytes = static_cast(sizeof(fsst_decoder_t)); + std::vector truncated(static_cast(header_bytes - 1), 0); + + EXPECT_THROW( + decoder->SetData(num_values, truncated.data(), static_cast(truncated.size())), + ParquetException); +} + +TEST(TestFsstEncoding, HeavyNullsDecodeSpaced) { + ::arrow::random::RandomArrayGenerator rag(321); + constexpr int64_t kNumValues = 4096; + constexpr double kNullProb = 0.85; + + auto values = rag.String(kNumValues, 1, 24, kNullProb); + + auto encoder = MakeTypedEncoder(Encoding::FSST); + ASSERT_NO_THROW(encoder->Put(*values)); + auto encoded = encoder->FlushValues(); + ASSERT_NE(nullptr, encoded); + + auto decoder = MakeTypedDecoder(Encoding::FSST); + decoder->SetData(static_cast(values->length()), encoded->data(), + static_cast(encoded->size())); + + const auto& binary = checked_cast(*values); + std::vector decoded(values->length()); + std::vector valid_bits(::arrow::bit_util::BytesForBits(values->length()), 0); + + ::arrow::internal::BitmapWriter writer(valid_bits.data(), 0, values->length()); + for (int64_t i = 0; i < values->length(); ++i) { + if (!binary.IsNull(i)) { + writer.Set(); + } else { + writer.Clear(); + } + writer.Next(); + } + writer.Finish(); + + const int null_count = binary.null_count(); + ASSERT_EQ(static_cast(values->length() - null_count), + decoder->DecodeSpaced(decoded.data(), static_cast(values->length()), + null_count, valid_bits.data(), 0)); + + for (int64_t i = 0; i < values->length(); ++i) { + if (binary.IsNull(i)) { + EXPECT_EQ(nullptr, decoded[i].ptr); + EXPECT_EQ(0u, decoded[i].len); + continue; + } + auto view = binary.GetView(i); + ASSERT_EQ(view.size(), static_cast(decoded[i].len)) << i; + ASSERT_EQ(0, memcmp(decoded[i].ptr, view.data(), view.size())) << i; + } +} + } // namespace parquet::test diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift index e3cc5adb9648..b94868ec3782 100644 --- a/cpp/src/parquet/parquet.thrift +++ b/cpp/src/parquet/parquet.thrift @@ -629,6 +629,13 @@ enum Encoding { Support for INT32, INT64 and FIXED_LEN_BYTE_ARRAY added in 2.11. */ BYTE_STREAM_SPLIT = 9; + + /** Fast Static Symbol Table (FSST) encoding for BYTE_ARRAY data. + FSST compresses strings using a symbol table that maps 1-8 byte sequences + to single-byte codes. It allows random access to compressed data and works + well with dictionary encoding by compressing dictionary values. + */ + FSST = 10; } /** diff --git a/cpp/src/parquet/thirdparty/fsst/fsst.cpp b/cpp/src/parquet/thirdparty/fsst/fsst.cpp new file mode 100644 index 000000000000..c1a9cc349803 --- /dev/null +++ b/cpp/src/parquet/thirdparty/fsst/fsst.cpp @@ -0,0 +1,200 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2019, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +#ifdef FSST12 +#include "fsst12.h" // the official FSST API -- also usable by C mortals +#else +#include "fsst.h" // the official FSST API -- also usable by C mortals +#endif +#include +#include +#include +#include +#include +#include +using namespace std; + +// Utility to compress and decompress (-d) data with FSST (using stdin and stdout). +// +// The utility has a poor-man's async I/O in that it uses double buffering for input and output, +// and two background pthreads for reading and writing. The idea is to make the CPU overlap with I/O. +// +// The data format is quite simple. A FSST compressed file is a sequence of blocks, each with format: +// (1) 3-byte block length field (max blocksize is hence 16MB). This byte-length includes (1), (2) and (3). +// (2) FSST dictionary as produced by fst_export(). +// (3) the FSST compressed data. +// +// The natural strength of FSST is in fact not block-based compression, but rather the compression and +// *individual* decompression of many small strings separately. Think of compressed databases and (column-store) +// data formats. But, this utility is to serve as an apples-to-apples comparison point with utilities like lz4. + +namespace { + +class BinarySemaphore { + private: + mutex m; + condition_variable cv; + bool value; + + public: + explicit BinarySemaphore(bool initialValue = false) : value(initialValue) {} + void wait() { + unique_lock lock(m); + while (!value) cv.wait(lock); + value = false; + } + void post() { + { unique_lock lock(m); value = true; } + cv.notify_one(); + } +}; + +bool stopThreads = false; +BinarySemaphore srcDoneIO[2], dstDoneIO[2], srcDoneCPU[2], dstDoneCPU[2]; +unsigned char *srcBuf[2] = { NULL, NULL }; +unsigned char *dstBuf[2] = { NULL, NULL }; +unsigned char *dstMem[2] = { NULL, NULL }; +size_t srcLen[2] = { 0, 0 }; +size_t dstLen[2] = { 0, 0 }; + +#define FSST_MEMBUF (1ULL<<22) +int decompress = 0; +size_t blksz = FSST_MEMBUF-(1+FSST_MAXHEADER/2); // block size of compression (max compressed size must fit 3 bytes) + +#define DESERIALIZE(p) (((unsigned long long) (p)[0]) << 16) | (((unsigned long long) (p)[1]) << 8) | ((unsigned long long) (p)[2]) +#define SERIALIZE(l,p) { (p)[0] = ((l)>>16)&255; (p)[1] = ((l)>>8)&255; (p)[2] = (l)&255; } + +void reader(ifstream& src) { + for(int swap=0; true; swap = 1-swap) { + srcDoneCPU[swap].wait(); + if (stopThreads) break; + src.read((char*) srcBuf[swap], blksz); + srcLen[swap] = (unsigned long) src.gcount(); + if (decompress) { + if (blksz && srcLen[swap] == blksz) { + blksz = DESERIALIZE(srcBuf[swap]+blksz-3); // read size of next block + srcLen[swap] -= 3; // cut off size bytes + } else { + blksz = 0; + } + } + srcDoneIO[swap].post(); + } +} + +void writer(ofstream& dst) { + for(int swap=0; true; swap = 1-swap) { + dstDoneCPU[swap].wait(); + if (!dstLen[swap]) break; + dst.write((char*) dstBuf[swap], dstLen[swap]); + dstDoneIO[swap].post(); + } + for(int swap=0; swap<2; swap++) + dstDoneIO[swap].post(); +} + +} + +#ifdef FSST_STANDALONE +int main(int argc, char* argv[]) { + size_t srcTot = 0, dstTot = 0; + if (argc < 2 || argc > 4 || (argc == 4 && (argv[1][0] != '-' || argv[1][1] != 'd' || argv[1][2]))) { + cerr << "usage: " << argv[0] << " -d infile outfile" << endl; + cerr << " " << argv[0] << " infile outfile" << endl; + cerr << " " << argv[0] << " infile" << endl; + return -1; + } + decompress = (argc == 4); + string srcfile(argv[1+decompress]), dstfile; + if (argc == 2) { + dstfile = srcfile + ".fsst"; + } else { + dstfile = argv[2+decompress]; + } + ifstream src; + ofstream dst; + src.open(srcfile, ios::binary); + dst.open(dstfile, ios::binary); + dst.exceptions(ios_base::failbit); + dst.exceptions(ios_base::badbit); + src.exceptions(ios_base::badbit); + if (decompress) { + unsigned char tmp[3]; + src.read((char*) tmp, 3); + if (src.gcount() != 3) { + cerr << "failed to open input." << endl; + return -1; + } + blksz = DESERIALIZE(tmp); // read first block size + } + vector buffer(FSST_MEMBUF*6); + srcBuf[0] = buffer.data(); + srcBuf[1] = srcBuf[0] + (FSST_MEMBUF*(1ULL+decompress)); + dstMem[0] = srcBuf[1] + (FSST_MEMBUF*(1ULL+decompress)); + dstMem[1] = dstMem[0] + (FSST_MEMBUF*(2ULL-decompress)); + + for(int swap=0; swap<2; swap++) { + srcDoneCPU[swap].post(); // input buffer is not being processed initially + dstDoneIO[swap].post(); // output buffer is not being written initially + } + thread readerThread([&src]{ reader(src); }); + thread writerThread([&dst]{ writer(dst); }); + + for(int swap=0; true; swap = 1-swap) { + srcDoneIO[swap].wait(); // wait until input buffer is available (i.e. done reading) + dstDoneIO[swap].wait(); // wait until output buffer is ready writing hence free for use + if (srcLen[swap] == 0) { + dstLen[swap] = 0; + break; + } + if (decompress) { + fsst_decoder_t decoder; + size_t hdr = fsst_import(&decoder, srcBuf[swap]); + dstLen[swap] = fsst_decompress(&decoder, srcLen[swap] - hdr, srcBuf[swap] + hdr, FSST_MEMBUF, dstBuf[swap] = dstMem[swap]); + } else { + unsigned char tmp[FSST_MAXHEADER]; + fsst_encoder_t* encoder = fsst_create(1, &srcLen[swap], const_cast(&srcBuf[swap]), 0); + size_t hdr = fsst_export(encoder, tmp); + if (fsst_compress(encoder, 1, &srcLen[swap], const_cast(&srcBuf[swap]), + FSST_MEMBUF * 2, dstMem[swap] + FSST_MAXHEADER + 3, + &dstLen[swap], &dstBuf[swap]) < 1) + return -1; + dstLen[swap] += 3 + hdr; + dstBuf[swap] -= 3 + hdr; + SERIALIZE(dstLen[swap],dstBuf[swap]); // block starts with size + copy(tmp, tmp+hdr, dstBuf[swap]+3); // then the header (followed by the compressed bytes which are already there) + fsst_destroy(encoder); + } + srcTot += srcLen[swap]; + dstTot += dstLen[swap]; + srcDoneCPU[swap].post(); // input buffer may be re-used by the reader for the next block + dstDoneCPU[swap].post(); // output buffer is ready for writing out + } + cerr << (decompress?"Dec":"C") << "ompressed " << srcTot << " bytes into " << dstTot << " bytes ==> " << (int) ((100*dstTot)/srcTot) << "%" << endl; + + // force wait until all background writes finished + stopThreads = true; + for(int swap=0; swap<2; swap++) { + srcDoneCPU[swap].post(); + dstDoneCPU[swap].post(); + } + dstDoneIO[0].wait(); + dstDoneIO[1].wait(); + readerThread.join(); + writerThread.join(); +} +#endif // FSST_STANDALONE diff --git a/cpp/src/parquet/thirdparty/fsst/fsst.h b/cpp/src/parquet/thirdparty/fsst/fsst.h new file mode 100644 index 000000000000..71085d57201d --- /dev/null +++ b/cpp/src/parquet/thirdparty/fsst/fsst.h @@ -0,0 +1,227 @@ +/* + * the API for FSST compression -- (c) Peter Boncz, Viktor Leis and Thomas Neumann (CWI, TU Munich), 2018-2019 + * + * =================================================================================================================================== + * this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): + * + * Copyright 2018-2020, CWI, TU Munich, FSU Jena + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + * IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * You can contact the authors via the FSST source repository : https://github.com/cwida/fsst + * =================================================================================================================================== + * + * FSST: Fast Static Symbol Table compression + * see the paper https://github.com/cwida/fsst/raw/master/fsstcompression.pdf + * + * FSST is a compression scheme focused on string/text data: it can compress strings from distributions with many different values (i.e. + * where dictionary compression will not work well). It allows *random-access* to compressed data: it is not block-based, so individual + * strings can be decompressed without touching the surrounding data in a compressed block. When compared to e.g. lz4 (which is + * block-based), FSST achieves similar decompression speed, (2x) better compression speed and 30% better compression ratio on text. + * + * FSST encodes strings also using a symbol table -- but it works on pieces of the string, as it maps "symbols" (1-8 byte sequences) + * onto "codes" (single-bytes). FSST can also represent a byte as an exception (255 followed by the original byte). Hence, compression + * transforms a sequence of bytes into a (supposedly shorter) sequence of codes or escaped bytes. These shorter byte-sequences could + * be seen as strings again and fit in whatever your program is that manipulates strings. + * + * useful property: FSST ensures that strings that are equal, are also equal in their compressed form. + * + * In this API, strings are considered byte-arrays (byte = unsigned char) and a batch of strings is represented as an array of + * unsigned char* pointers to their starts. A seperate length array (of unsigned int) denotes how many bytes each string consists of. + * + * This representation as unsigned char* pointers tries to assume as little as possible on the memory management of the program + * that calls this API, and is also intended to allow passing strings into this API without copying (even if you use C++ strings). + * + * We optionally support C-style zero-terminated strings (zero appearing only at the end). In this case, the compressed strings are + * also zero-terminated strings. In zero-terminated mode, the zero-byte at the end *is* counted in the string byte-length. + */ +#ifndef FSST_INCLUDED_H +#define FSST_INCLUDED_H + +#ifdef _MSC_VER +#define __restrict__ +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ +#define __ORDER_LITTLE_ENDIAN__ 2 +#include +static inline int __builtin_ctzl(unsigned long long x) { + unsigned long ret; + _BitScanForward64(&ret, x); + return (int)ret; +} +#endif + +#ifdef __cplusplus +#define FSST_FALLTHROUGH [[fallthrough]] +#include +extern "C" { +#else +#define FSST_FALLTHROUGH +#endif + +#include + +/* A compressed string is simply a string of 1-byte codes; except for code 255, which is followed by an uncompressed byte. */ +#define FSST_ESC 255 + +/* Data structure needed for compressing strings - use fsst_duplicate() to create thread-local copies. Use fsst_destroy() to free. */ +typedef void* fsst_encoder_t; /* opaque type - it wraps around a rather large (~900KB) C++ object */ + +/* Data structure needed for decompressing strings - read-only and thus can be shared between multiple decompressing threads. */ +typedef struct { + unsigned long long version; /* version id */ + unsigned char zeroTerminated; /* terminator is a single-byte code that does not appear in longer symbols */ + unsigned char len[255]; /* len[x] is the byte-length of the symbol x (1 < len[x] <= 8). */ + unsigned long long symbol[255]; /* symbol[x] contains in LITTLE_ENDIAN the bytesequence that code x represents (0 <= x < 255). */ +} fsst_decoder_t; + +/* Calibrate a FSST symboltable from a batch of strings (it is best to provide at least 16KB of data). */ +fsst_encoder_t* +fsst_create( + size_t n, /* IN: number of strings in batch to sample from. */ + const size_t lenIn[], /* IN: byte-lengths of the inputs */ + const unsigned char *strIn[], /* IN: string start pointers. */ + int zeroTerminated /* IN: whether input strings are zero-terminated. If so, encoded strings are as well (i.e. symbol[0]=""). */ +); + +/* Create another encoder instance, necessary to do multi-threaded encoding using the same symbol table. */ +fsst_encoder_t* +fsst_duplicate( + fsst_encoder_t *encoder /* IN: the symbol table to duplicate. */ +); + +#define FSST_MAXHEADER (8+1+8+2048+1) /* maxlen of deserialized fsst header, produced/consumed by fsst_export() resp. fsst_import() */ + +/* Space-efficient symbol table serialization (smaller than sizeof(fsst_decoder_t) - by saving on the unused bytes in symbols of len < 8). */ +unsigned int /* OUT: number of bytes written in buf, at most sizeof(fsst_decoder_t) */ +fsst_export( + fsst_encoder_t *encoder, /* IN: the symbol table to dump. */ + unsigned char *buf /* OUT: pointer to a byte-buffer where to serialize this symbol table. */ +); + +/* Deallocate encoder. */ +void +fsst_destroy(fsst_encoder_t*); + +/* Return a decoder structure from serialized format (typically used in a block-, file- or row-group header). */ +unsigned int /* OUT: number of bytes consumed in buf (0 on failure). */ +fsst_import( + fsst_decoder_t *decoder, /* IN: this symbol table will be overwritten. */ + unsigned char const *buf /* IN: pointer to a byte-buffer where fsst_export() serialized this symbol table. */ +); + +/* Return a decoder structure from an encoder. */ +fsst_decoder_t +fsst_decoder( + fsst_encoder_t *encoder +); + +/* Compress a batch of strings (on AVX512 machines best performance is obtained by compressing more than 32KB of string volume). */ +/* The output buffer must be large; at least "conservative space" (7+2*inputlength) for the first string for something to happen. */ +size_t /* OUT: the number of compressed strings (<=n) that fit the output buffer. */ +fsst_compress( + fsst_encoder_t *encoder, /* IN: encoder obtained from fsst_create(). */ + size_t nstrings, /* IN: number of strings in batch to compress. */ + const size_t lenIn[], /* IN: byte-lengths of the inputs */ + const unsigned char *strIn[], /* IN: input string start pointers. */ + size_t outsize, /* IN: byte-length of output buffer. */ + unsigned char *output, /* OUT: memory buffer to put the compressed strings in (one after the other). */ + size_t lenOut[], /* OUT: byte-lengths of the compressed strings. */ + unsigned char *strOut[] /* OUT: output string start pointers. Will all point into [output,output+size). */ +); + +/* Decompress a single string, inlined for speed. */ +inline size_t /* OUT: bytesize of the decompressed string. If > size, the decoded output is truncated to size. */ +fsst_decompress( + const fsst_decoder_t *decoder, /* IN: use this symbol table for compression. */ + size_t lenIn, /* IN: byte-length of compressed string. */ + const unsigned char *strIn, /* IN: compressed string. */ + size_t size, /* IN: byte-length of output buffer. */ + unsigned char *output /* OUT: memory buffer to put the decompressed string in. */ +) { + unsigned char*__restrict__ len = (unsigned char* __restrict__) decoder->len; + unsigned char*__restrict__ strOut = (unsigned char* __restrict__) output; + unsigned long long*__restrict__ symbol = (unsigned long long* __restrict__) decoder->symbol; + size_t code, posOut = 0, posIn = 0; +#ifndef FSST_MUST_ALIGN /* defining on platforms that require aligned memory access may help their performance */ +#define FSST_UNALIGNED_STORE(dst,src) memcpy((unsigned long long*) (dst), &(src), sizeof(unsigned long long)) +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + while (posOut+32 <= size && posIn+4 <= lenIn) { + unsigned int nextBlock, escapeMask; + memcpy(&nextBlock, strIn+posIn, sizeof(unsigned int)); + escapeMask = (nextBlock&0x80808080u)&((((~nextBlock)&0x7F7F7F7Fu)+0x7F7F7F7Fu)^0x80808080u); + if (escapeMask == 0) { + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + } else { + unsigned long firstEscapePos=__builtin_ctzl((unsigned long long) escapeMask)>>3; + switch(firstEscapePos) { /* Duff's device */ + case 3: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + // fall through + case 2: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + // fall through + case 1: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + // fall through + case 0: posIn+=2; strOut[posOut++] = strIn[posIn-1]; /* decompress an escaped byte */ + } + } + } + if (posOut+32 <= size) { // handle the possibly 3 last bytes without a loop + if (posIn+2 <= lenIn) { + strOut[posOut] = strIn[posIn+1]; + if (strIn[posIn] != FSST_ESC) { + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + if (strIn[posIn] != FSST_ESC) { + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + } else { + posIn += 2; strOut[posOut++] = strIn[posIn-1]; + } + } else { + posIn += 2; posOut++; + } + } + if (posIn < lenIn) { // last code cannot be an escape + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + } + } +#else + while (posOut+8 <= size && posIn < lenIn) + if ((code = strIn[posIn++]) < FSST_ESC) { /* symbol compressed as code? */ + FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); /* unaligned memory write */ + posOut += len[code]; + } else { + strOut[posOut] = strIn[posIn]; /* decompress an escaped byte */ + posIn++; posOut++; + } +#endif +#endif + while (posIn < lenIn) + if ((code = strIn[posIn++]) < FSST_ESC) { + size_t posWrite = posOut, endWrite = posOut + len[code]; + unsigned char* __restrict__ symbolPointer = ((unsigned char* __restrict__) &symbol[code]) - posWrite; + if ((posOut = endWrite) > size) endWrite = size; + for(; posWrite < endWrite; posWrite++) /* only write if there is room */ + strOut[posWrite] = symbolPointer[posWrite]; + } else { + if (posOut < size) strOut[posOut] = strIn[posIn]; /* idem */ + posIn++; posOut++; + } + if (posOut >= size && (decoder->zeroTerminated&1)) strOut[size-1] = 0; + return posOut; /* full size of decompressed string (could be >size, then the actually decompressed part) */ +} + +#ifdef __cplusplus +} +#endif +#endif /* FSST_INCLUDED_H */ diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512.cpp b/cpp/src/parquet/thirdparty/fsst/fsst_avx512.cpp new file mode 100644 index 000000000000..e43d3e03652c --- /dev/null +++ b/cpp/src/parquet/thirdparty/fsst/fsst_avx512.cpp @@ -0,0 +1,149 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +#include "libfsst.hpp" + +#if defined(__x86_64__) || defined(_M_X64) +#include + +#ifdef _WIN32 +namespace libfsst { +bool fsst_hasAVX512() { + int info[4]; + __cpuidex(info, 0x00000007, 0); + return (info[1]>>16)&1; +} +} // namespace libfsst +#else +#include +namespace libfsst { +bool fsst_hasAVX512() { + int info[4]; + __cpuid_count(0x00000007, 0, info[0], info[1], info[2], info[3]); + return (info[1]>>16)&1; +} +} // namespace libfsst +#endif +#else +namespace libfsst { +bool fsst_hasAVX512() { return false; } +} // namespace libfsst +#endif + +namespace libfsst { + +// BULK COMPRESSION OF STRINGS +// +// In one call of this function, we can compress 512 strings, each of maximum length 511 bytes. +// strings can be shorter than 511 bytes, no problem, but if they are longer we need to cut them up. +// +// In each iteration of the while loop, we find one code in each of the unroll*8 strings, i.e. (8,16,24 or 32) for resp. unroll=1,2,3,4 +// unroll3 performs best on my hardware +// +// In the worst case, each final encoded string occupies 512KB bytes (512*1024; with 1024=512xexception, exception = 2 bytes). +// - hence codeBase is a buffer of 512KB (needs 19 bits jobs), symbolBase of 256KB (needs 18 bits jobs). +// +// 'jobX' controls the encoding of each string and is therefore a u64 with format [out:19][pos:9][end:18][cur:18] (low-to-high bits) +// The field 'pos' tells which string we are processing (0..511). We need this info as strings will complete compressing out-of-order. +// +// Strings will have different lengths, and when a string is finished, we reload from the buffer of 512 input strings. +// This continues until we have less than (8,16,24 or 32; depending on unroll) strings left to process. +// - so 'processed' is the amount of strings we started processing and it is between [480,512]. +// Note that when we quit, there will still be some (<32) strings that we started to process but which are unfinished. +// - so 'unfinished' is that amount. These unfinished strings will be encoded further using the scalar method. +// +// Apart from the coded strings, we return in a output[] array of size 'processed' the job values of the 'finished' strings. +// In the following 'unfinished' slots (processed=finished+unfinished) we output the 'job' values of the unfinished strings. +// +// For the finished strings, we need [out:19] to see the compressed size and [pos:9] to see which string we refer to. +// For the unfinished strings, we need all fields of 'job' to continue the compression with scalar code (see SIMD code in compressBatch). +// +// THIS IS A SEPARATE CODE FILE NOT BECAUSE OF MY LOVE FOR MODULARIZED CODE BUT BECAUSE IT ALLOWS TO COMPILE IT WITH DIFFERENT FLAGS +// in particular, unrolling is crucial for gather/scatter performance, but requires registers. the #define all_* expressions however, +// will be detected to be constants by g++ -O2 and will be precomputed and placed into AVX512 registers - spoiling 9 of them. +// This reduces the effectiveness of unrolling, hence -O2 makes the loop perform worse than -O1 which skips this optimization. +// Assembly inspection confirmed that 3-way unroll with -O1 avoids needless load/stores. + +size_t fsst_compressAVX512(SymbolTable &symbolTable, u8* codeBase, u8* symbolBase, SIMDjob *input, SIMDjob *output, size_t n, size_t unroll) { + size_t processed = 0; + // define some constants (all_x means that all 8 lanes contain 64-bits value X) +#ifdef __AVX512F__ + //__m512i all_suffixLim= _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) symbolTable->suffixLim)); -- for variants b,c + __m512i all_MASK = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) -1)); + __m512i all_PRIME = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) FSST_HASH_PRIME)); + __m512i all_ICL_FREE = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) FSST_ICL_FREE)); +#define all_HASH _mm512_srli_epi64(all_MASK, 64-FSST_HASH_LOG2SIZE) +#define all_ONE _mm512_srli_epi64(all_MASK, 63) +#define all_M19 _mm512_srli_epi64(all_MASK, 45) +#define all_M18 _mm512_srli_epi64(all_MASK, 46) +#define all_M28 _mm512_srli_epi64(all_MASK, 36) +#define all_FFFFFF _mm512_srli_epi64(all_MASK, 40) +#define all_FFFF _mm512_srli_epi64(all_MASK, 48) +#define all_FF _mm512_srli_epi64(all_MASK, 56) + + SIMDjob *inputEnd = input+n; + assert(n >= unroll*8 && n <= 512); // should be close to 512 + __m512i job1, job2, job3, job4; // will contain current jobs, for each unroll 1,2,3,4 + __mmask8 loadmask1 = 255, loadmask2 = 255*(unroll>1), loadmask3 = 255*(unroll>2), loadmask4 = 255*(unroll>3); // 2b loaded new strings bitmask per unroll + u32 delta1 = 8, delta2 = 8*(unroll>1), delta3 = 8*(unroll>2), delta4 = 8*(unroll>3); // #new loads this SIMD iteration per unroll + + if (unroll >= 4) { + while (input+delta1+delta2+delta3+delta4 < inputEnd) { + #include "fsst_avx512_unroll4.inc" + } + } else if (unroll == 3) { + while (input+delta1+delta2+delta3 < inputEnd) { + #include "fsst_avx512_unroll3.inc" + } + } else if (unroll == 2) { + while (input+delta1+delta2 < inputEnd) { + #include "fsst_avx512_unroll2.inc" + } + } else { + while (input+delta1 < inputEnd) { + #include "fsst_avx512_unroll1.inc" + } + } + + // flush the job states of the unfinished strings at the end of output[] + processed = n - (inputEnd - input); + u32 unfinished = 0; + if (unroll > 1) { + if (unroll > 2) { + if (unroll > 3) { + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask4=~loadmask4, job4); + unfinished += _mm_popcnt_u32((int) loadmask4); + } + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask3=~loadmask3, job3); + unfinished += _mm_popcnt_u32((int) loadmask3); + } + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask2=~loadmask2, job2); + unfinished += _mm_popcnt_u32((int) loadmask2); + } + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask1=~loadmask1, job1); +#else + (void) symbolTable; + (void) codeBase; + (void) symbolBase; + (void) input; + (void) output; + (void) n; + (void) unroll; +#endif + return processed; +} +} // namespace libfsst diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512.inc b/cpp/src/parquet/thirdparty/fsst/fsst_avx512.inc new file mode 100644 index 000000000000..0a74541dd884 --- /dev/null +++ b/cpp/src/parquet/thirdparty/fsst/fsst_avx512.inc @@ -0,0 +1,57 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmaskX=11111111, deltaX=8). + jobX = _mm512_mask_expandloadu_epi64(jobX, loadmaskX, input); input += deltaX; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i wordX = _mm512_i64gather_epi64(_mm512_srli_epi64(jobX, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // codeX: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i codeX = _mm512_i64gather_epi64(_mm512_and_epi64(wordX, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + __m512i posX = _mm512_mullo_epi64(_mm512_and_epi64(wordX, all_FFFFFF), all_PRIME); + // hash them into a random number: posX = posX*PRIME; posX ^= posX>>SHIFT + posX = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(posX,_mm512_srli_epi64(posX,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + __m512i iclX = _mm512_i64gather_epi64(posX, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the writeX register (in case it turns out to be an escaped byte). + __m512i writeX = _mm512_slli_epi64(_mm512_and_epi64(wordX, all_FF), 8); + // lookup just like the iclX above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symbX = _mm512_i64gather_epi64(posX, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + posX = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(iclX, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 matchX = _mm512_cmpeq_epi64_mask(symbX, _mm512_and_epi64(wordX, posX)) & _mm512_cmplt_epi64_mask(iclX, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + codeX = _mm512_mask_mov_epi64(codeX, matchX, _mm512_srli_epi64(iclX, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + writeX = _mm512_or_epi64(writeX, _mm512_and_epi64(codeX, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + codeX = _mm512_and_epi64(codeX, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(jobX, all_M19), writeX, 1); + // increase the jobX.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + jobX = _mm512_add_epi64(jobX, _mm512_slli_epi64(_mm512_srli_epi64(codeX, FSST_LEN_BITS), 46)); + // increase the jobX.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + jobX = _mm512_add_epi64(jobX, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(codeX, 8), all_ONE))); + // test which lanes are done now (jobX.cur==jobX.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the jobX register) + loadmaskX = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(jobX, 46), _mm512_and_epi64(_mm512_srli_epi64(jobX, 28), all_M18)); + // calculate the amount of lanes in jobX that are done + deltaX = _mm_popcnt_u32((int) loadmaskX); + // write out the job state for the lanes that are done (we need the final 'jobX.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmaskX, jobX); output += deltaX; diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll1.inc b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll1.inc new file mode 100644 index 000000000000..f4b81c7970dd --- /dev/null +++ b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll1.inc @@ -0,0 +1,57 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll2.inc b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll2.inc new file mode 100644 index 000000000000..aa33cd7e69c5 --- /dev/null +++ b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll2.inc @@ -0,0 +1,114 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + code2 = _mm512_and_epi64(code2, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + // calculate the amount of lanes in job2 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + delta2 = _mm_popcnt_u32((int) loadmask2); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; + _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll3.inc b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll3.inc new file mode 100644 index 000000000000..e2057032abd3 --- /dev/null +++ b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll3.inc @@ -0,0 +1,171 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// +// +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E3PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// +// +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask3=11111111, delta3=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; + job3 = _mm512_mask_expandloadu_epi64(job3, loadmask3, input); input += delta3; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); + __m512i word3 = _mm512_i64gather_epi64(_mm512_srli_epi64(job3, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code3: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code3 = _mm512_i64gather_epi64(_mm512_and_epi64(word3, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + // get the first three bytes of the string. + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); + __m512i pos3 = _mm512_mullo_epi64(_mm512_and_epi64(word3, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT + // hash them into a random number: pos3 = pos3*PRIME; pos3 ^= pos3>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); + pos3 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos3,_mm512_srli_epi64(pos3,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write3 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); + __m512i write3 = _mm512_slli_epi64(_mm512_and_epi64(word3, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl3 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); + pos3 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl3, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); + __mmask8 match3 = _mm512_cmpeq_epi64_mask(symb3, _mm512_and_epi64(word3, pos3)) & _mm512_cmplt_epi64_mask(icl3, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); + code3 = _mm512_mask_mov_epi64(code3, match3, _mm512_srli_epi64(icl3, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); + write3 = _mm512_or_epi64(write3, _mm512_and_epi64(code3, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + code2 = _mm512_and_epi64(code2, all_FFFF); + code3 = _mm512_and_epi64(code3, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job3, all_M19), write3, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job3.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); + job3 = _mm512_add_epi64(job3, _mm512_slli_epi64(_mm512_srli_epi64(code3, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job3.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); + job3 = _mm512_add_epi64(job3, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code3, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) + // test which lanes are done now (job3.cur==job3.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job3 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); + loadmask3 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job3, 46), _mm512_and_epi64(_mm512_srli_epi64(job3, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + // calculate the amount of lanes in job2 that are done + // calculate the amount of lanes in job3 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + delta2 = _mm_popcnt_u32((int) loadmask2); + delta3 = _mm_popcnt_u32((int) loadmask3); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job3.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; + _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; + _mm512_mask_compressstoreu_epi64(output, loadmask3, job3); output += delta3; diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll4.inc b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll4.inc new file mode 100644 index 000000000000..15cca7c938b4 --- /dev/null +++ b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll4.inc @@ -0,0 +1,228 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// +// +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// +// +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// +// +// +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// +// +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E3PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E4PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// +// +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// +// +// +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask3=11111111, delta3=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask4=11111111, delta4=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; + job3 = _mm512_mask_expandloadu_epi64(job3, loadmask3, input); input += delta3; + job4 = _mm512_mask_expandloadu_epi64(job4, loadmask4, input); input += delta4; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); + __m512i word3 = _mm512_i64gather_epi64(_mm512_srli_epi64(job3, 46), symbolBase, 1); + __m512i word4 = _mm512_i64gather_epi64(_mm512_srli_epi64(job4, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code3: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code4: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code3 = _mm512_i64gather_epi64(_mm512_and_epi64(word3, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code4 = _mm512_i64gather_epi64(_mm512_and_epi64(word4, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + // get the first three bytes of the string. + // get the first three bytes of the string. + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); + __m512i pos3 = _mm512_mullo_epi64(_mm512_and_epi64(word3, all_FFFFFF), all_PRIME); + __m512i pos4 = _mm512_mullo_epi64(_mm512_and_epi64(word4, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT + // hash them into a random number: pos3 = pos3*PRIME; pos3 ^= pos3>>SHIFT + // hash them into a random number: pos4 = pos4*PRIME; pos4 ^= pos4>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); + pos3 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos3,_mm512_srli_epi64(pos3,FSST_SHIFT)), all_HASH), 4); + pos4 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos4,_mm512_srli_epi64(pos4,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl4 = _mm512_i64gather_epi64(pos4, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write3 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write4 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); + __m512i write3 = _mm512_slli_epi64(_mm512_and_epi64(word3, all_FF), 8); + __m512i write4 = _mm512_slli_epi64(_mm512_and_epi64(word4, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl3 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl4 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb4 = _mm512_i64gather_epi64(pos4, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); + pos3 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl3, all_FF)); + pos4 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl4, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); + __mmask8 match3 = _mm512_cmpeq_epi64_mask(symb3, _mm512_and_epi64(word3, pos3)) & _mm512_cmplt_epi64_mask(icl3, all_ICL_FREE); + __mmask8 match4 = _mm512_cmpeq_epi64_mask(symb4, _mm512_and_epi64(word4, pos4)) & _mm512_cmplt_epi64_mask(icl4, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); + code3 = _mm512_mask_mov_epi64(code3, match3, _mm512_srli_epi64(icl3, 16)); + code4 = _mm512_mask_mov_epi64(code4, match4, _mm512_srli_epi64(icl4, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); + write3 = _mm512_or_epi64(write3, _mm512_and_epi64(code3, all_FF)); + write4 = _mm512_or_epi64(write4, _mm512_and_epi64(code4, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + code2 = _mm512_and_epi64(code2, all_FFFF); + code3 = _mm512_and_epi64(code3, all_FFFF); + code4 = _mm512_and_epi64(code4, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job3, all_M19), write3, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job4, all_M19), write4, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job3.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job4.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); + job3 = _mm512_add_epi64(job3, _mm512_slli_epi64(_mm512_srli_epi64(code3, FSST_LEN_BITS), 46)); + job4 = _mm512_add_epi64(job4, _mm512_slli_epi64(_mm512_srli_epi64(code4, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job3.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job4.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); + job3 = _mm512_add_epi64(job3, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code3, 8), all_ONE))); + job4 = _mm512_add_epi64(job4, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code4, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) + // test which lanes are done now (job3.cur==job3.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job3 register) + // test which lanes are done now (job4.cur==job4.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job4 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); + loadmask3 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job3, 46), _mm512_and_epi64(_mm512_srli_epi64(job3, 28), all_M18)); + loadmask4 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job4, 46), _mm512_and_epi64(_mm512_srli_epi64(job4, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + // calculate the amount of lanes in job2 that are done + // calculate the amount of lanes in job3 that are done + // calculate the amount of lanes in job4 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + delta2 = _mm_popcnt_u32((int) loadmask2); + delta3 = _mm_popcnt_u32((int) loadmask3); + delta4 = _mm_popcnt_u32((int) loadmask4); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job3.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job4.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; + _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; + _mm512_mask_compressstoreu_epi64(output, loadmask3, job3); output += delta3; + _mm512_mask_compressstoreu_epi64(output, loadmask4, job4); output += delta4; diff --git a/cpp/src/parquet/thirdparty/fsst/libfsst.cpp b/cpp/src/parquet/thirdparty/fsst/libfsst.cpp new file mode 100644 index 000000000000..e3ba787b9592 --- /dev/null +++ b/cpp/src/parquet/thirdparty/fsst/libfsst.cpp @@ -0,0 +1,651 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +#include "libfsst.hpp" + +namespace libfsst { +Symbol concat(Symbol a, Symbol b) { + Symbol s; + u32 length = a.length()+b.length(); + if (length > Symbol::maxLength) length = Symbol::maxLength; + s.set_code_len(FSST_CODE_MASK, length); + s.store_num((b.load_num() << (8*a.length())) | a.load_num()); + return s; +} +} // namespace libfsst + +namespace std { +template <> +class hash { + public: + size_t operator()(const libfsst::QSymbol& q) const { + uint64_t k = q.symbol.load_num(); + const uint64_t m = 0xc6a4a7935bd1e995; + const int r = 47; + uint64_t h = 0x8445d61a4e774912 ^ (8*m); + k *= m; + k ^= k >> r; + k *= m; + h ^= k; + h *= m; + h ^= h >> r; + h *= m; + h ^= h >> r; + return h; + } +}; +} + +namespace libfsst { +bool isEscapeCode(u16 pos) { return pos < FSST_CODE_BASE; } + +std::ostream& operator<<(std::ostream& out, const Symbol& s) { + for (u32 i=0; i line, const size_t len[], bool zeroTerminated=false) { + SymbolTable *st = new SymbolTable(), *bestTable = new SymbolTable(); + int bestGain = (int) -FSST_SAMPLEMAXSZ; // worst case (everything exception) + size_t sampleFrac = 128; + + // start by determining the terminator. We use the (lowest) most infrequent byte as terminator + st->zeroTerminated = zeroTerminated; + if (zeroTerminated) { + st->terminator = 0; // except in case of zeroTerminated mode, then byte 0 is terminator regardless frequency + } else { + u16 byteHisto[256]; + memset(byteHisto, 0, sizeof(byteHisto)); + for(size_t i=0; iterminator = 256; + while(i-- > 0) { + if (byteHisto[i] > minSize) continue; + st->terminator = i; + minSize = byteHisto[i]; + } + } + assert(st->terminator != 256); + + // a random number between 0 and 128 + auto rnd128 = [&](size_t i) { return 1 + (FSST_HASH((i+1UL)*sampleFrac)&127); }; + + // compress sample, and compute (pair-)frequencies + auto compressCount = [&](SymbolTable *st, Counters &counters) { // returns gain + int gain = 0; + + for(size_t i=0; i sampleFrac) continue; + } + if (cur < end) { + u16 code2 = 255, code1 = st->findLongestSymbol(cur, end); + cur += st->symbols[code1].length(); + gain += (int) (st->symbols[code1].length()-(1+isEscapeCode(code1))); + while (true) { + // count single symbol (i.e. an option is not extending it) + counters.count1Inc(code1); + + // as an alternative, consider just using the next byte.. + if (st->symbols[code1].length() != 1) // .. but do not count single byte symbols doubly + counters.count1Inc(*start); + + if (cur==end) { + break; + } + + // now match a new symbol + start = cur; + if (curhashTabSize-1); + Symbol s = st->hashTab[idx]; + code2 = st->shortCodes[word & 0xFFFF] & FSST_CODE_MASK; + word &= (0xFFFFFFFFFFFFFFFF >> (u8) s.icl); + if ((s.icl < FSST_ICL_FREE) & (s.load_num() == word)) { + code2 = s.code(); + cur += s.length(); + } else if (code2 >= FSST_CODE_BASE) { + cur += 2; + } else { + code2 = st->byteCodes[word & 0xFF] & FSST_CODE_MASK; + cur += 1; + } + } else { + code2 = st->findLongestSymbol(cur, end); + cur += st->symbols[code2].length(); + } + + // compute compressed output size + gain += ((int) (cur-start))-(1+isEscapeCode(code2)); + + if (sampleFrac < 128) { // no need to count pairs in final round + // consider the symbol that is the concatenation of the two last symbols + counters.count2Inc(code1, code2); + + // as an alternative, consider just extending with the next byte.. + if ((cur-start) > 1) // ..but do not count single byte extensions doubly + counters.count2Inc(code1, *start); + } + code1 = code2; + } + } + } + return gain; + }; + + auto makeTable = [&](SymbolTable *st, Counters &counters) { + // hashmap of c (needed because we can generate duplicate candidates) + unordered_set cands; + + // artificially make terminater the most frequent symbol so it gets included + u16 terminator = st->nSymbols?FSST_CODE_BASE:st->terminator; + counters.count1Set(terminator,65535); + + auto addOrInc = [&](unordered_set &cands, Symbol s, u64 count) { + if (count < (5*sampleFrac)/128) return; // improves both compression speed (less candidates), but also quality!! + QSymbol q; + q.symbol = s; + q.gain = count * s.length(); + auto it = cands.find(q); + if (it != cands.end()) { + q.gain += (*it).gain; + cands.erase(*it); + } + cands.insert(q); + }; + + // add candidate symbols based on counted frequency + for (u32 pos1=0; pos1nSymbols; pos1++) { + u32 cnt1 = counters.count1GetNext(pos1); // may advance pos1!! + if (!cnt1) continue; + + // heuristic: promoting single-byte symbols (*8) helps reduce exception rates and increases [de]compression speed + Symbol s1 = st->symbols[pos1]; + addOrInc(cands, s1, ((s1.length()==1)?8LL:1LL)*cnt1); + + if (sampleFrac >= 128 || // last round we do not create new (combined) symbols + s1.length() == Symbol::maxLength || // symbol cannot be extended + s1.val.str[0] == st->terminator) { // multi-byte symbols cannot contain the terminator byte + continue; + } + for (u32 pos2=0; pos2nSymbols; pos2++) { + u32 cnt2 = counters.count2GetNext(pos1, pos2); // may advance pos2!! + if (!cnt2) continue; + + // create a new symbol + Symbol s2 = st->symbols[pos2]; + Symbol s3 = concat(s1, s2); + if (s2.val.str[0] != st->terminator) // multi-byte symbols cannot contain the terminator byte + addOrInc(cands, s3, cnt2); + } + } + + // insert candidates into priority queue (by gain) + auto cmpGn = [](const QSymbol& q1, const QSymbol& q2) { return (q1.gain < q2.gain) || (q1.gain == q2.gain && q1.symbol.load_num() > q2.symbol.load_num()); }; + priority_queue,decltype(cmpGn)> pq(cmpGn); + for (auto& q : cands) + pq.push(q); + + // Create new symbol map using best candidates + st->clear(); + while (st->nSymbols < 255 && !pq.empty()) { + QSymbol q = pq.top(); + pq.pop(); + st->add(q.symbol); + } + }; + + u8 bestCounters[512*sizeof(u16)]; +#ifdef NONOPT_FSST + for(size_t frac : {127, 127, 127, 127, 127, 127, 127, 127, 127, 128}) { + sampleFrac = frac; +#else + for(sampleFrac=8; true; sampleFrac += 30) { +#endif + memset(&counters, 0, sizeof(Counters)); + long gain = compressCount(st, counters); + if (gain >= bestGain) { // a new best solution! + counters.backup1(bestCounters); + *bestTable = *st; bestGain = gain; + } + if (sampleFrac >= 128) break; // we do 5 rounds (sampleFrac=8,38,68,98,128) + makeTable(st, counters); + } + delete st; + counters.restore1(bestCounters); + makeTable(bestTable, counters); + bestTable->finalize(zeroTerminated); // renumber codes for more efficient compression + return bestTable; +} + +#ifndef NONOPT_FSST +static inline size_t compressSIMD(SymbolTable &symbolTable, u8* symbolBase, size_t nlines, const size_t len[], const u8* line[], size_t size, u8* dst, size_t lenOut[], u8* strOut[], int unroll) { + size_t curLine = 0, inOff = 0, outOff = 0, batchPos = 0, empty = 0, budget = size; + u8 *lim = dst + size, *codeBase = symbolBase + (1<<18); // 512KB temp space for compressing 512 strings + SIMDjob input[512]; // combined offsets of input strings (cur,end), and string #id (pos) and output (dst) pointer + SIMDjob output[512]; // output are (pos:9,dst:19) end pointers (compute compressed length from this) + size_t jobLine[512]; // for which line in the input sequence was this job (needed because we may split a line into multiple jobs) + + while (curLine < nlines && outOff <= (1<<19)) { + size_t prevLine = curLine, chunk, curOff = 0; + + // bail out if the output buffer cannot hold the compressed next string fully + if (((len[curLine]-curOff)*2 + 7) > budget) break; // see below for the +7 + else budget -= (len[curLine]-curOff)*2; + + strOut[curLine] = (u8*) 0; + lenOut[curLine] = 0; + + do { + do { + chunk = len[curLine] - curOff; + if (chunk > 511) { + chunk = 511; // large strings need to be chopped up into segments of 511 bytes + } + // create a job in this batch + SIMDjob job; + job.cur = inOff; + job.end = job.cur + chunk; + job.pos = batchPos; + job.out = outOff; + + // worst case estimate for compressed size (+7 is for the scatter that writes extra 7 zeros) + outOff += 7 + 2*(size_t)(job.end - job.cur); // note, total size needed is 512*(511*2+7) bytes. + if (outOff > (1<<19)) break; // simdbuf may get full, stop before this chunk + + // register job in this batch + input[batchPos] = job; + jobLine[batchPos] = curLine; + + if (chunk == 0) { + empty++; // detect empty chunks -- SIMD code cannot handle empty strings, so they need to be filtered out + } else { + // copy string chunk into temp buffer + memcpy(symbolBase + inOff, line[curLine] + curOff, chunk); + inOff += chunk; + curOff += chunk; + symbolBase[inOff++] = (u8) symbolTable.terminator; // write an extra char at the end that will not be encoded + } + if (++batchPos == 512) break; + } while(curOff < len[curLine]); + + if ((batchPos == 512) || (outOff > (1<<19)) || (++curLine >= nlines) || (((len[curLine])*2 + 7) > budget)) { // cannot accumulate more? + if (batchPos-empty >= 32) { // if we have enough work, fire off fsst_compressAVX512 (32 is due to max 4x8 unrolling) + // radix-sort jobs on length (longest string first) + // -- this provides best load balancing and allows to skip empty jobs at the end + u16 sortpos[513]; + memset(sortpos, 0, sizeof(sortpos)); + + // calculate length histo + for(size_t i=0; i> (u8) s.icl); + if ((s.icl < FSST_ICL_FREE) && s.load_num() == word) { + *out++ = (u8) s.code(); cur += s.length(); + } else { + // could be a 2-byte or 1-byte code, or miss + // handle everything with predication + *out = (u8) code; + out += 1+((code&FSST_CODE_BASE)>>8); + cur += (code>>FSST_LEN_BITS); + } + } + job.out = out - codeBase; + } + // postprocess job info + job.cur = 0; + job.end = job.out - input[job.pos].out; // misuse .end field as compressed size + job.out = input[job.pos].out; // reset offset to start of encoded string + input[job.pos] = job; + } + + // copy out the result data + for(size_t i=0; i> (u8) s.icl); + if ((s.icl < FSST_ICL_FREE) && s.load_num() == word) { + *out++ = (u8) s.code(); cur += s.length(); + } else if (avoidBranch) { + // could be a 2-byte or 1-byte code, or miss + // handle everything with predication + *out = (u8) code; + out += 1+((code&FSST_CODE_BASE)>>8); + cur += (code>>FSST_LEN_BITS); + } else if ((u8) code < byteLim) { + // 2 byte code after checking there is no longer pattern + *out++ = (u8) code; cur += 2; + } else { + // 1 byte code or miss. + *out = (u8) code; + out += 1+((code&FSST_CODE_BASE)>>8); // predicated - tested with a branch, that was always worse + cur++; + } + } + } + }; + + for(curLine=0; curLine 511) { + chunk = 511; // we need to compress in chunks of 511 in order to be byte-compatible with simd-compressed FSST + } + if ((2*chunk+7) > (size_t) (lim-out)) { + return curLine; // out of memory + } + // copy the string to the 511-byte buffer + memcpy(buf, cur, chunk); + buf[chunk] = (u8) symbolTable.terminator; + cur = buf; + end = cur + chunk; + + // based on symboltable stats, choose a variant that is nice to the branch predictor + if (noSuffixOpt) { + compressVariant(true,false); + } else if (avoidBranch) { + compressVariant(false,true); + } else { + compressVariant(false, false); + } + } while((curOff += chunk) < lenIn[curLine]); + lenOut[curLine] = (size_t) (out - strOut[curLine]); + } + return curLine; +} + +#define FSST_SAMPLELINE ((size_t) 512) + +// quickly select a uniformly random set of lines such that we have between [FSST_SAMPLETARGET,FSST_SAMPLEMAXSZ) string bytes +vector makeSample(u8* sampleBuf, const u8* strIn[], const size_t **lenRef, size_t nlines) { + size_t totSize = 0; + const size_t *lenIn = *lenRef; + vector sample; + + for(size_t i=0; i sample = makeSample(sampleBuf, strIn, &sampleLen, n?n:1); // careful handling of input to get a right-size and representative sample + Encoder *encoder = new Encoder(); + encoder->symbolTable = shared_ptr(buildSymbolTable(encoder->counters, sample, sampleLen, zeroTerminated)); + if (sampleLen != lenIn) delete[] sampleLen; + delete[] sampleBuf; + return (fsst_encoder_t*) encoder; +} + +/* create another encoder instance, necessary to do multi-threaded encoding using the same symbol table */ +extern "C" fsst_encoder_t* fsst_duplicate(fsst_encoder_t *encoder) { + Encoder *e = new Encoder(); + e->symbolTable = ((Encoder*)encoder)->symbolTable; // it is a shared_ptr + return (fsst_encoder_t*) e; +} + +// export a symbol table in compact format. +extern "C" u32 fsst_export(fsst_encoder_t *encoder, u8 *buf) { + Encoder *e = (Encoder*) encoder; + // In ->version there is a versionnr, but we hide also suffixLim/terminator/nSymbols there. + // This is sufficient in principle to *reconstruct* a fsst_encoder_t from a fsst_decoder_t + // (such functionality could be useful to append compressed data to an existing block). + // + // However, the hash function in the encoder hash table is endian-sensitive, and given its + // 'lossy perfect' hashing scheme is *unable* to contain other-endian-produced symbol tables. + // Doing a endian-conversion during hashing will be slow and self-defeating. + // + // Overall, we could support reconstructing an encoder for incremental compression, but + // should enforce equal-endianness. Bit of a bummer. Not going there now. + // + // The version field is now there just for future-proofness, but not used yet + + // version allows keeping track of fsst versions, track endianness, and encoder reconstruction + u64 version = (FSST_VERSION << 32) | // version is 24 bits, most significant byte is 0 + (((u64) e->symbolTable->suffixLim) << 24) | + (((u64) e->symbolTable->terminator) << 16) | + (((u64) e->symbolTable->nSymbols) << 8) | + FSST_ENDIAN_MARKER; // least significant byte is nonzero + + version = swap64_if_be(version); // ensure version is little-endian encoded + + /* do not assume unaligned reads here */ + memcpy(buf, &version, 8); + buf[8] = e->symbolTable->zeroTerminated; + for(u32 i=0; i<8; i++) + buf[9+i] = (u8) e->symbolTable->lenHisto[i]; + u32 pos = 17; + + // emit only the used bytes of the symbols + for(u32 i = e->symbolTable->zeroTerminated; i < e->symbolTable->nSymbols; i++) + for(u32 j = 0; j < e->symbolTable->symbols[i].length(); j++) + buf[pos++] = e->symbolTable->symbols[i].val.str[j]; // serialize used symbol bytes + + return pos; // length of what was serialized +} + +#define FSST_CORRUPT 32774747032022883 /* 7-byte number in little endian containing "corrupt" */ + +extern "C" u32 fsst_import(fsst_decoder_t *decoder, u8 const *buf) { + u64 version = 0; + u32 code, pos = 17; + u8 lenHisto[8]; + + // version field (first 8 bytes) is now there just for future-proofness, unused still (skipped) + memcpy(&version, buf, 8); + version = swap64_if_be(version); // version is always little-endian encoded + + if ((version>>32) != FSST_VERSION) return 0; + decoder->zeroTerminated = buf[8]&1; + memcpy(lenHisto, buf+9, 8); + + // in case of zero-terminated, first symbol is "" (zero always, may be overwritten) + decoder->len[0] = 1; + decoder->symbol[0] = 0; + + // we use lenHisto[0] as 1-byte symbol run length (at the end) + code = decoder->zeroTerminated; + if (decoder->zeroTerminated) lenHisto[0]--; // if zeroTerminated, then symbol "" aka 1-byte code=0, is not stored at the end + + // now get all symbols from the buffer + for(u32 l=1; l<=8; l++) { /* l = 1,2,3,4,5,6,7,8 */ + for(u32 i=0; i < lenHisto[(l&7) /* 1,2,3,4,5,6,7,0 */]; i++, code++) { + decoder->len[code] = (l&7)+1; /* len = 2,3,4,5,6,7,8,1 */ + decoder->symbol[code] = 0; + for(u32 j=0; jlen[code]; j++) + ((u8*) &decoder->symbol[code])[j] = buf[pos++]; // note this enforces 'little endian' symbols + } + } + if (decoder->zeroTerminated) lenHisto[0]++; + + // fill unused symbols with text "corrupt". Gives a chance to detect corrupted code sequences (if there are unused symbols). + while(code<255) { + decoder->symbol[code] = FSST_CORRUPT; + decoder->len[code++] = 8; + } + return pos; +} + +// runtime check for simd +inline size_t _compressImpl(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd) { +#ifndef NONOPT_FSST + if (simd && fsst_hasAVX512()) + return compressSIMD(*e->symbolTable, e->simdbuf, nlines, lenIn, strIn, size, output, lenOut, strOut, simd); +#endif + (void) simd; + return compressBulk(*e->symbolTable, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch); +} +size_t compressImpl(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd) { + return _compressImpl(e, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch, simd); +} + +// adaptive choosing of scalar compression method based on symbol length histogram +inline size_t _compressAuto(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], int simd) { + bool avoidBranch = false, noSuffixOpt = false; + if (100*e->symbolTable->lenHisto[1] > 65*e->symbolTable->nSymbols && 100*e->symbolTable->suffixLim > 95*e->symbolTable->lenHisto[1]) { + noSuffixOpt = true; + } else if ((e->symbolTable->lenHisto[0] > 24 && e->symbolTable->lenHisto[0] < 92) && + (e->symbolTable->lenHisto[0] < 43 || e->symbolTable->lenHisto[6] + e->symbolTable->lenHisto[7] < 29) && + (e->symbolTable->lenHisto[0] < 72 || e->symbolTable->lenHisto[2] < 72)) { + avoidBranch = true; + } + return _compressImpl(e, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch, simd); +} +size_t compressAuto(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], int simd) { + return _compressAuto(e, nlines, lenIn, strIn, size, output, lenOut, strOut, simd); +} +} // namespace libfsst + +using namespace libfsst; +// the main compression function (everything automatic) +extern "C" size_t fsst_compress(fsst_encoder_t *encoder, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[]) { + // to be faster than scalar, simd needs 64 lines or more of length >=12; or fewer lines, but big ones (totLen > 32KB) + size_t totLen = accumulate(lenIn, lenIn+nlines, 0); + int simd = totLen > nlines*12 && (nlines > 64 || totLen > (size_t) 1<<15); + return _compressAuto((Encoder*) encoder, nlines, lenIn, strIn, size, output, lenOut, strOut, 3*simd); +} + +/* deallocate encoder */ +extern "C" void fsst_destroy(fsst_encoder_t* encoder) { + Encoder *e = (Encoder*) encoder; + delete e; +} + +/* very lazy implementation relying on export and import */ +extern "C" fsst_decoder_t fsst_decoder(fsst_encoder_t *encoder) { + u8 buf[sizeof(fsst_decoder_t)]; + u32 cnt1 = fsst_export(encoder, buf); + fsst_decoder_t decoder; + u32 cnt2 = fsst_import(&decoder, buf); + assert(cnt1 == cnt2); (void) cnt1; (void) cnt2; + return decoder; +} diff --git a/cpp/src/parquet/thirdparty/fsst/libfsst.hpp b/cpp/src/parquet/thirdparty/fsst/libfsst.hpp new file mode 100644 index 000000000000..f61bc0175b63 --- /dev/null +++ b/cpp/src/parquet/thirdparty/fsst/libfsst.hpp @@ -0,0 +1,471 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +#include "fsst.h" // the official FSST API -- also usable by C mortals + +/* unsigned integers */ +namespace libfsst { +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +} // namespace libfsst + +#if UINTPTR_MAX == 0xffffffffU +// We're on a 32-bit platform +#define NONOPT_FSST +#endif + +#define FSST_ENDIAN_MARKER ((u64) 1) +#define FSST_VERSION_20190218 20190218 +#define FSST_VERSION ((u64) FSST_VERSION_20190218) + +// "symbols" are character sequences (up to 8 bytes) +// A symbol is compressed into a "code" of, in principle, one byte. But, we added an exception mechanism: +// byte 255 followed by byte X represents the single-byte symbol X. Its code is 256+X. + +// we represent codes in u16 (not u8). 12 bits code (of which 10 are used), 4 bits length +#define FSST_LEN_BITS 12 +#define FSST_CODE_BITS 9 +#define FSST_CODE_BASE 256UL /* first 256 codes [0,255] are pseudo codes: escaped bytes */ +#define FSST_CODE_MAX (1UL<=8) { + len = 8; + memcpy(val.str, input, 8); + } else { + memcpy(val.str, input, len); + } + set_code_len(FSST_CODE_MAX, len); + } + void set_code_len(u32 code, u32 len) { icl = (len<<28)|(code<<16)|((8-len)*8); } + + u64 load_num() const { return swap64_if_be(val.num); } + void store_num(u64 v) { val.num = swap64_if_be(v); } + + u32 length() const { return (u32) (icl >> 28); } + u16 code() const { return (icl >> 16) & FSST_CODE_MASK; } + u32 ignoredBits() const { return (u32) icl; } + + u8 first() const { assert( length() >= 1); return 0xFF & load_num(); } + u16 first2() const { assert( length() >= 2); return 0xFFFF & load_num(); } + +#define FSST_HASH_LOG2SIZE 10 +#define FSST_HASH_PRIME 2971215073LL +#define FSST_SHIFT 15 +#define FSST_HASH(w) (((w)*FSST_HASH_PRIME)^(((w)*FSST_HASH_PRIME)>>FSST_SHIFT)) + size_t hash() const { size_t v = 0xFFFFFF & load_num(); return FSST_HASH(v); } // hash on the next 3 bytes +}; + +// Symbol that can be put in a queue, ordered on gain +struct QSymbol{ + Symbol symbol; + mutable u32 gain; // mutable because gain value should be ignored in find() on unordered_set of QSymbols + bool operator==(const QSymbol& other) const { return symbol.val.num == other.symbol.val.num && symbol.length() == other.symbol.length(); } +}; + +// we construct FSST symbol tables using a random sample of about 16KB (1<<14) +#define FSST_SAMPLETARGET (1<<14) +#define FSST_SAMPLEMAXSZ ((long) 2*FSST_SAMPLETARGET) + +// two phases of compression, before and after optimize(): +// +// (1) to encode values we probe (and maintain) three datastructures: +// - u16 byteCodes[256] array at the position of the next byte (s.length==1) +// - u16 shortCodes[65536] array at the position of the next twobyte pattern (s.length==2) +// - Symbol hashtable[1024] (keyed by the next three bytes, ie for s.length>2), +// this search will yield a u16 code, it points into Symbol symbols[]. You always find a hit, because the first 256 codes are +// pseudo codes representing a single byte these will become escapes) +// +// (2) when we finished looking for the best symbol table we call optimize() to reshape it: +// - it renumbers the codes by length (first symbols of length 2,3,4,5,6,7,8; then 1 (starting from byteLim are symbols of length 1) +// length 2 codes for which no longer suffix symbol exists (< suffixLim) come first among the 2-byte codes +// (allows shortcut during compression) +// - for each two-byte combination, in all unused slots of shortCodes[], it enters the byteCode[] of the symbol corresponding +// to the first byte (if such a single-byte symbol exists). This allows us to just probe the next two bytes (if there is only one +// byte left in the string, there is still a terminator-byte added during compression) in shortCodes[]. That is, byteCodes[] +// and its codepath is no longer required. This makes compression faster. The reason we use byteCodes[] during symbolTable construction +// is that adding a new code/symbol is expensive (you have to touch shortCodes[] in 256 places). This optimization was +// hence added to make symbolTable construction faster. +// +// this final layout allows for the fastest compression code, only currently present in compressBulk + +// in the hash table, the icl field contains (low-to-high) ignoredBits:16,code:12,length:4 +#define FSST_ICL_FREE ((15<<28)|(((u32)FSST_CODE_MASK)<<16)) // high bits of icl (len=8,code=FSST_CODE_MASK) indicates free bucket + +// ignoredBits is (8-length)*8, which is the amount of high bits to zero in the input word before comparing with the hashtable key +// ..it could of course be computed from len during lookup, but storing it precomputed in some loose bits is faster +// +// the gain field is only used in the symbol queue that sorts symbols on gain + +struct SymbolTable { + static const u32 hashTabSize = 1<> (u8) s.icl)); + return true; + } + bool add(Symbol s) { + assert(FSST_CODE_BASE + nSymbols < FSST_CODE_MAX); + u32 len = s.length(); + s.set_code_len(FSST_CODE_BASE + nSymbols, len); + if (len == 1) { + byteCodes[s.first()] = FSST_CODE_BASE + nSymbols + (1<> ((u8) hashTab[idx].icl)))) { + return (hashTab[idx].icl>>16) & FSST_CODE_MASK; // matched a long symbol + } + if (s.length() >= 2) { + u16 code = shortCodes[s.first2()] & FSST_CODE_MASK; + if (code >= FSST_CODE_BASE) return code; + } + return byteCodes[s.first()] & FSST_CODE_MASK; + } + u16 findLongestSymbol(const u8* cur, const u8* end) const { + return findLongestSymbol(Symbol(cur,end)); // represent the string as a temporary symbol + } + + // rationale for finalize: + // - during symbol table construction, we may create more than 256 codes, but bring it down to max 255 in the last makeTable() + // consequently we needed more than 8 bits during symbol table contruction, but can simplify the codes to single bytes in finalize() + // (this feature is in fact lo longer used, but could still be exploited: symbol construction creates no more than 255 symbols in each pass) + // - we not only reduce the amount of codes to <255, but also *reorder* the symbols and renumber their codes, for higher compression perf. + // we renumber codes so they are grouped by length, to allow optimized scalar string compression (byteLim and suffixLim optimizations). + // - we make the use of byteCode[] no longer necessary by inserting single-byte codes in the free spots of shortCodes[] + // Using shortCodes[] only makes compression faster. When creating the symbolTable, however, using shortCodes[] for the single-byte + // symbols is slow, as each insert touches 256 positions in it. This optimization was added when optimizing symbolTable construction time. + // + // In all, we change the layout and coding, as follows.. + // + // before finalize(): + // - The real symbols are symbols[256..256+nSymbols>. As we may have nSymbols > 255 + // - The first 256 codes are pseudo symbols (all escaped bytes) + // + // after finalize(): + // - table layout is symbols[0..nSymbols>, with nSymbols < 256. + // - Real codes are [0,nSymbols>. 8-th bit not set. + // - Escapes in shortCodes have the 8th bit set (value: 256+255=511). 255 because the code to be emitted is the escape byte 255 + // - symbols are grouped by length: 2,3,4,5,6,7,8, then 1 (single-byte codes last) + // the two-byte codes are split in two sections: + // - first section contains codes for symbols for which there is no longer symbol (no suffix). It allows an early-out during compression + // + // finally, shortCodes[] is modified to also encode all single-byte symbols (hence byteCodes[] is not required on a critical path anymore). + // + void finalize(u8 zeroTerminated) { + assert(nSymbols <= 255); + u8 newCode[256], rsum[8], byteLim = nSymbols - (lenHisto[0] - zeroTerminated); + + // compute running sum of code lengths (starting offsets for each length) + rsum[0] = byteLim; // 1-byte codes are highest + rsum[1] = zeroTerminated; + for(u32 i=1; i<7; i++) + rsum[i+1] = rsum[i] + lenHisto[i]; + + // determine the new code for each symbol, ordered by length (and splitting 2byte symbols into two classes around suffixLim) + suffixLim = rsum[1]; + symbols[newCode[0] = 0] = symbols[256]; // keep symbol 0 in place (for zeroTerminated cases only) + + for(u32 i=zeroTerminated, j=rsum[2]; i 1 && first2 == s2.first2()) // test if symbol k is a suffix of s + opt = 0; + } + newCode[i] = opt?suffixLim++:--j; // symbols without a larger suffix have a code < suffixLim + } else + newCode[i] = rsum[len-1]++; + s1.set_code_len(newCode[i],len); + symbols[newCode[i]] = s1; + } + // renumber the codes in byteCodes[] + for(u32 i=0; i<256; i++) + if ((byteCodes[i] & FSST_CODE_MASK) >= FSST_CODE_BASE) + byteCodes[i] = newCode[(u8) byteCodes[i]] + (1 << FSST_LEN_BITS); + else + byteCodes[i] = 511 + (1 << FSST_LEN_BITS); + + // renumber the codes in shortCodes[] + for(u32 i=0; i<65536; i++) + if ((shortCodes[i] & FSST_CODE_MASK) >= FSST_CODE_BASE) + shortCodes[i] = newCode[(u8) shortCodes[i]] + (shortCodes[i] & (15 << FSST_LEN_BITS)); + else + shortCodes[i] = byteCodes[i&0xFF]; + + // replace the symbols in the hash table + for(u32 i=0; i>8; + } + void count1Inc(u32 pos1) { + if (!count1Low[pos1]++) // increment high early (when low==0, not when low==255). This means (high > 0) <=> (cnt > 0) + count1High[pos1]++; //(0,0)->(1,1)->..->(255,1)->(0,1)->(1,2)->(2,2)->(3,2)..(255,2)->(0,2)->(1,3)->(2,3)... + } + void count2Inc(u32 pos1, u32 pos2) { + if (!count2Low[pos1][pos2]++) // increment high early (when low==0, not when low==255). This means (high > 0) <=> (cnt > 0) + // inc 4-bits high counter with 1<<0 (1) or 1<<4 (16) -- depending on whether pos2 is even or odd, repectively + count2High[pos1][(pos2)>>1] += 1 << (((pos2)&1)<<2); // we take our chances with overflow.. (4K maxval, on a 8K sample) + } + u32 count1GetNext(u32 &pos1) { // note: we will advance pos1 to the next nonzero counter in register range + // read 16-bits single symbol counter, split into two 8-bits numbers (count1Low, count1High), while skipping over zeros + u64 high = fsst_unaligned_load(&count1High[pos1]); // note: this reads 8 subsequent counters [pos1..pos1+7] + + u32 zero = high?(__builtin_ctzl(high)>>3):7UL; // number of zero bytes + high = (high >> (zero << 3)) & 255; // advance to nonzero counter + if (((pos1 += zero) >= FSST_CODE_MAX) || !high) // SKIP! advance pos2 + return 0; // all zero + + u32 low = count1Low[pos1]; + if (low) high--; // high is incremented early and low late, so decrement high (unless low==0) + return (u32) ((high << 8) + low); + } + u32 count2GetNext(u32 pos1, u32 &pos2) { // note: we will advance pos2 to the next nonzero counter in register range + // read 12-bits pairwise symbol counter, split into low 8-bits and high 4-bits number while skipping over zeros + u64 high = fsst_unaligned_load(&count2High[pos1][pos2>>1]); // note: this reads 16 subsequent counters [pos2..pos2+15] + high >>= ((pos2&1) << 2); // odd pos2: ignore the lowest 4 bits & we see only 15 counters + + u32 zero = high?(__builtin_ctzl(high)>>2):(15UL-(pos2&1UL)); // number of zero 4-bits counters + high = (high >> (zero << 2)) & 15; // advance to nonzero counter + if (((pos2 += zero) >= FSST_CODE_MAX) || !high) // SKIP! advance pos2 + return 0UL; // all zero + + u32 low = count2Low[pos1][pos2]; + if (low) high--; // high is incremented early and low late, so decrement high (unless low==0) + return (u32) ((high << 8) + low); + } + void backup1(u8 *buf) { + memcpy(buf, count1High, FSST_CODE_MAX); + memcpy(buf+FSST_CODE_MAX, count1Low, FSST_CODE_MAX); + } + void restore1(u8 *buf) { + memcpy(count1High, buf, FSST_CODE_MAX); + memcpy(count1Low, buf+FSST_CODE_MAX, FSST_CODE_MAX); + } +}; +#endif + + +#define FSST_BUFSZ (3<<19) // 768KB + +// an encoder is a symbolmap plus some bufferspace, needed during map construction as well as compression +struct Encoder { + shared_ptr symbolTable; // symbols, plus metadata and data structures for quick compression (shortCode,hashTab, etc) + union { + Counters counters; // for counting symbol occurences during map construction + u8 simdbuf[FSST_BUFSZ]; // for compression: SIMD string staging area 768KB = 256KB in + 512KB out (worst case for 256KB in) + }; +}; + +// job control integer representable in one 64bits SIMD lane: cur/end=input, out=output, pos=which string (2^9=512 per call) +struct SIMDjob { + u64 out:19,pos:9,end:18,cur:18; // cur/end is input offsets (2^18=256KB), out is output offset (2^19=512KB) +}; + +extern bool +fsst_hasAVX512(); // runtime check for avx512 capability + +extern size_t +fsst_compressAVX512( + SymbolTable &symbolTable, + u8* codeBase, // IN: base address for codes, i.e. compression output (points to simdbuf+256KB) + u8* symbolBase, // IN: base address for string bytes, i.e. compression input (points to simdbuf) + SIMDjob* input, // IN: input array (size n) with job information: what to encode, where to store it. + SIMDjob* output, // OUT: output array (size n) with job information: how much got encoded, end output pointer. + size_t n, // IN: size of arrays input and output (should be max 512) + size_t unroll); // IN: degree of SIMD unrolling + +// C++ fsst-compress function with some more control of how the compression happens (algorithm flavor, simd unroll degree) +size_t compressImpl(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd); +size_t compressAuto(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], int simd); +} // namespace libfsst diff --git a/cpp/src/parquet/types.cc b/cpp/src/parquet/types.cc index 7109c3d1c83b..7e5324814b0d 100644 --- a/cpp/src/parquet/types.cc +++ b/cpp/src/parquet/types.cc @@ -259,6 +259,8 @@ std::string EncodingToString(Encoding::type t) { return "RLE_DICTIONARY"; case Encoding::BYTE_STREAM_SPLIT: return "BYTE_STREAM_SPLIT"; + case Encoding::FSST: + return "FSST"; default: return "UNKNOWN"; } diff --git a/cpp/src/parquet/types.h b/cpp/src/parquet/types.h index c2040e555fdb..0113ec86aa13 100644 --- a/cpp/src/parquet/types.h +++ b/cpp/src/parquet/types.h @@ -531,8 +531,9 @@ struct Encoding { DELTA_BYTE_ARRAY = 7, RLE_DICTIONARY = 8, BYTE_STREAM_SPLIT = 9, + FSST = 10, // Should always be last element (except UNKNOWN) - UNDEFINED = 10, + UNDEFINED = 11, UNKNOWN = 999 }; }; diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 14cd3e363a46..86f9f27305a0 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -1487,6 +1487,7 @@ cdef encoding_name_from_enum(ParquetEncoding encoding_): ParquetEncoding_DELTA_BYTE_ARRAY: 'DELTA_BYTE_ARRAY', ParquetEncoding_RLE_DICTIONARY: 'RLE_DICTIONARY', ParquetEncoding_BYTE_STREAM_SPLIT: 'BYTE_STREAM_SPLIT', + ParquetEncoding_FSST: 'FSST', }.get(encoding_, 'UNKNOWN') @@ -1499,6 +1500,7 @@ cdef encoding_enum_from_name(str encoding_name): 'DELTA_BINARY_PACKED': ParquetEncoding_DELTA_BINARY_PACKED, 'DELTA_LENGTH_BYTE_ARRAY': ParquetEncoding_DELTA_LENGTH_BYTE_ARRAY, 'DELTA_BYTE_ARRAY': ParquetEncoding_DELTA_BYTE_ARRAY, + 'FSST': ParquetEncoding_FSST, 'RLE_DICTIONARY': 'dict', 'PLAIN_DICTIONARY': 'dict', }.get(encoding_name, None) diff --git a/python/pyarrow/includes/libparquet.pxd b/python/pyarrow/includes/libparquet.pxd index 42d48ba050f1..3663b8bcf62b 100644 --- a/python/pyarrow/includes/libparquet.pxd +++ b/python/pyarrow/includes/libparquet.pxd @@ -130,6 +130,7 @@ cdef extern from "parquet/api/schema.h" namespace "parquet" nogil: ParquetEncoding_RLE_DICTIONARY" parquet::Encoding::RLE_DICTIONARY" ParquetEncoding_BYTE_STREAM_SPLIT \ " parquet::Encoding::BYTE_STREAM_SPLIT" + ParquetEncoding_FSST" parquet::Encoding::FSST" enum ParquetCompression" parquet::Compression::type": ParquetCompression_UNCOMPRESSED" parquet::Compression::UNCOMPRESSED" diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 24cb586c82b3..5fb3cc5b699c 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -821,7 +821,7 @@ def _sanitize_table(table, new_schema, flavor): Can only be used when ``use_dictionary`` is set to False, and cannot be used in combination with ``use_byte_stream_split``. Currently supported values: {'PLAIN', 'BYTE_STREAM_SPLIT', - 'DELTA_BINARY_PACKED', 'DELTA_LENGTH_BYTE_ARRAY', 'DELTA_BYTE_ARRAY'}. + 'DELTA_BINARY_PACKED', 'DELTA_LENGTH_BYTE_ARRAY', 'DELTA_BYTE_ARRAY', 'FSST'}. Certain encodings are only compatible with certain data types. Please refer to the encodings section of `Reading and writing Parquet files `_. From 9ba9672710837cee922a2c5aa7dea1b78230b9f8 Mon Sep 17 00:00:00 2001 From: arnavb Date: Sun, 23 Nov 2025 14:20:27 +0000 Subject: [PATCH 02/24] update --- cpp/src/parquet/thirdparty/fsst/libfsst.hpp | 127 ++++++++++---------- 1 file changed, 65 insertions(+), 62 deletions(-) diff --git a/cpp/src/parquet/thirdparty/fsst/libfsst.hpp b/cpp/src/parquet/thirdparty/fsst/libfsst.hpp index f61bc0175b63..e4d7c0aa9b9c 100644 --- a/cpp/src/parquet/thirdparty/fsst/libfsst.hpp +++ b/cpp/src/parquet/thirdparty/fsst/libfsst.hpp @@ -1,20 +1,20 @@ // this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// +// // Copyright 2018-2020, CWI, TU Munich, FSU Jena -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: -// +// // - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst #include #include #include @@ -59,7 +59,7 @@ typedef uint64_t u64; // we represent codes in u16 (not u8). 12 bits code (of which 10 are used), 4 bits length #define FSST_LEN_BITS 12 -#define FSST_CODE_BITS 9 +#define FSST_CODE_BITS 9 #define FSST_CODE_BASE 256UL /* first 256 codes [0,255] are pseudo codes: escaped bytes */ #define FSST_CODE_MAX (1UL<= 1); return 0xFF & load_num(); } u16 first2() const { assert( length() >= 2); return 0xFFFF & load_num(); } -#define FSST_HASH_LOG2SIZE 10 +#define FSST_HASH_LOG2SIZE 10 #define FSST_HASH_PRIME 2971215073LL #define FSST_SHIFT 15 #define FSST_HASH(w) (((w)*FSST_HASH_PRIME)^(((w)*FSST_HASH_PRIME)>>FSST_SHIFT)) @@ -129,7 +129,7 @@ struct QSymbol{ bool operator==(const QSymbol& other) const { return symbol.val.num == other.symbol.val.num && symbol.length() == other.symbol.length(); } }; -// we construct FSST symbol tables using a random sample of about 16KB (1<<14) +// we construct FSST symbol tables using a random sample of about 16KB (1<<14) #define FSST_SAMPLETARGET (1<<14) #define FSST_SAMPLEMAXSZ ((long) 2*FSST_SAMPLETARGET) @@ -138,15 +138,15 @@ struct QSymbol{ // (1) to encode values we probe (and maintain) three datastructures: // - u16 byteCodes[256] array at the position of the next byte (s.length==1) // - u16 shortCodes[65536] array at the position of the next twobyte pattern (s.length==2) -// - Symbol hashtable[1024] (keyed by the next three bytes, ie for s.length>2), -// this search will yield a u16 code, it points into Symbol symbols[]. You always find a hit, because the first 256 codes are +// - Symbol hashtable[1024] (keyed by the next three bytes, ie for s.length>2), +// this search will yield a u16 code, it points into Symbol symbols[]. You always find a hit, because the first 256 codes are // pseudo codes representing a single byte these will become escapes) // // (2) when we finished looking for the best symbol table we call optimize() to reshape it: // - it renumbers the codes by length (first symbols of length 2,3,4,5,6,7,8; then 1 (starting from byteLim are symbols of length 1) -// length 2 codes for which no longer suffix symbol exists (< suffixLim) come first among the 2-byte codes +// length 2 codes for which no longer suffix symbol exists (< suffixLim) come first among the 2-byte codes // (allows shortcut during compression) -// - for each two-byte combination, in all unused slots of shortCodes[], it enters the byteCode[] of the symbol corresponding +// - for each two-byte combination, in all unused slots of shortCodes[], it enters the byteCode[] of the symbol corresponding // to the first byte (if such a single-byte symbol exists). This allows us to just probe the next two bytes (if there is only one // byte left in the string, there is still a terminator-byte added during compression) in shortCodes[]. That is, byteCodes[] // and its codepath is no longer required. This makes compression faster. The reason we use byteCodes[] during symbolTable construction @@ -173,9 +173,9 @@ struct SymbolTable { u16 byteCodes[256]; // contains code for every 1-byte symbol, otherwise code for pseudo byte (escaped byte) // 'symbols' is the current symbol table symbol[code].symbol is the max 8-byte 'symbol' for single-byte 'code' - Symbol symbols[FSST_CODE_MAX]; // x in [0,255]: pseudo symbols representing escaped byte x; x in [FSST_CODE_BASE=256,256+nSymbols]: real symbols + Symbol symbols[FSST_CODE_MAX]; // x in [0,255]: pseudo symbols representing escaped byte x; x in [FSST_CODE_BASE=256,256+nSymbols]: real symbols - // replicate long symbols in hashTab (avoid indirection). + // replicate long symbols in hashTab (avoid indirection). Symbol hashTab[hashTabSize]; // used for all symbols of 3 and more bytes u16 nSymbols; // amount of symbols in the map (max 255) @@ -225,8 +225,8 @@ struct SymbolTable { u32 idx = symbols[i].hash() & (hashTabSize-1); hashTab[idx].val.num = 0; hashTab[idx].icl = FSST_ICL_FREE; //marks empty in hashtab - } - } + } + } nSymbols = 0; // no need to clean symbols[] as no symbols are used } bool hashInsert(Symbol s) { @@ -256,11 +256,11 @@ struct SymbolTable { u16 findLongestSymbol(Symbol s) const { size_t idx = s.hash() & (hashTabSize-1); if (hashTab[idx].icl <= s.icl && hashTab[idx].load_num() == (s.load_num() & (0xFFFFFFFFFFFFFFFF >> ((u8) hashTab[idx].icl)))) { - return (hashTab[idx].icl>>16) & FSST_CODE_MASK; // matched a long symbol + return (hashTab[idx].icl>>16) & FSST_CODE_MASK; // matched a long symbol } if (s.length() >= 2) { u16 code = shortCodes[s.first2()] & FSST_CODE_MASK; - if (code >= FSST_CODE_BASE) return code; + if (code >= FSST_CODE_BASE) return code; } return byteCodes[s.first()] & FSST_CODE_MASK; } @@ -273,23 +273,23 @@ struct SymbolTable { // consequently we needed more than 8 bits during symbol table contruction, but can simplify the codes to single bytes in finalize() // (this feature is in fact lo longer used, but could still be exploited: symbol construction creates no more than 255 symbols in each pass) // - we not only reduce the amount of codes to <255, but also *reorder* the symbols and renumber their codes, for higher compression perf. - // we renumber codes so they are grouped by length, to allow optimized scalar string compression (byteLim and suffixLim optimizations). + // we renumber codes so they are grouped by length, to allow optimized scalar string compression (byteLim and suffixLim optimizations). // - we make the use of byteCode[] no longer necessary by inserting single-byte codes in the free spots of shortCodes[] // Using shortCodes[] only makes compression faster. When creating the symbolTable, however, using shortCodes[] for the single-byte // symbols is slow, as each insert touches 256 positions in it. This optimization was added when optimizing symbolTable construction time. // // In all, we change the layout and coding, as follows.. // - // before finalize(): + // before finalize(): // - The real symbols are symbols[256..256+nSymbols>. As we may have nSymbols > 255 // - The first 256 codes are pseudo symbols (all escaped bytes) // - // after finalize(): - // - table layout is symbols[0..nSymbols>, with nSymbols < 256. - // - Real codes are [0,nSymbols>. 8-th bit not set. + // after finalize(): + // - table layout is symbols[0..nSymbols>, with nSymbols < 256. + // - Real codes are [0,nSymbols>. 8-th bit not set. // - Escapes in shortCodes have the 8th bit set (value: 256+255=511). 255 because the code to be emitted is the escape byte 255 // - symbols are grouped by length: 2,3,4,5,6,7,8, then 1 (single-byte codes last) - // the two-byte codes are split in two sections: + // the two-byte codes are split in two sections: // - first section contains codes for symbols for which there is no longer symbol (no suffix). It allows an early-out during compression // // finally, shortCodes[] is modified to also encode all single-byte symbols (hence byteCodes[] is not required on a critical path anymore). @@ -298,7 +298,7 @@ struct SymbolTable { assert(nSymbols <= 255); u8 newCode[256], rsum[8], byteLim = nSymbols - (lenHisto[0] - zeroTerminated); - // compute running sum of code lengths (starting offsets for each length) + // compute running sum of code lengths (starting offsets for each length) rsum[0] = byteLim; // 1-byte codes are highest rsum[1] = zeroTerminated; for(u32 i=1; i<7; i++) @@ -308,34 +308,34 @@ struct SymbolTable { suffixLim = rsum[1]; symbols[newCode[0] = 0] = symbols[256]; // keep symbol 0 in place (for zeroTerminated cases only) - for(u32 i=zeroTerminated, j=rsum[2]; i 1 && first2 == s2.first2()) // test if symbol k is a suffix of s opt = 0; } - newCode[i] = opt?suffixLim++:--j; // symbols without a larger suffix have a code < suffixLim - } else + newCode[i] = opt?suffixLim++:--j; // symbols without a larger suffix have a code < suffixLim + } else newCode[i] = rsum[len-1]++; s1.set_code_len(newCode[i],len); - symbols[newCode[i]] = s1; + symbols[newCode[i]] = s1; } - // renumber the codes in byteCodes[] - for(u32 i=0; i<256; i++) + // renumber the codes in byteCodes[] + for(u32 i=0; i<256; i++) if ((byteCodes[i] & FSST_CODE_MASK) >= FSST_CODE_BASE) byteCodes[i] = newCode[(u8) byteCodes[i]] + (1 << FSST_LEN_BITS); - else + else byteCodes[i] = 511 + (1 << FSST_LEN_BITS); - - // renumber the codes in shortCodes[] + + // renumber the codes in shortCodes[] for(u32 i=0; i<65536; i++) if ((shortCodes[i] & FSST_CODE_MASK) >= FSST_CODE_BASE) shortCodes[i] = newCode[(u8) shortCodes[i]] + (shortCodes[i] & (15 << FSST_LEN_BITS)); - else + else shortCodes[i] = byteCodes[i&0xFF]; // replace the symbols in the hash table @@ -347,22 +347,22 @@ struct SymbolTable { #ifdef NONOPT_FSST struct Counters { - u16 count1[FSST_CODE_MAX]; // array to count frequency of symbols as they occur in the sample - u16 count2[FSST_CODE_MAX][FSST_CODE_MAX]; // array to count subsequent combinations of two symbols in the sample + u16 count1[FSST_CODE_MAX]; // array to count frequency of symbols as they occur in the sample + u16 count2[FSST_CODE_MAX][FSST_CODE_MAX]; // array to count subsequent combinations of two symbols in the sample - void count1Set(u32 pos1, u16 val) { + void count1Set(u32 pos1, u16 val) { count1[pos1] = val; } - void count1Inc(u32 pos1) { + void count1Inc(u32 pos1) { count1[pos1]++; } - void count2Inc(u32 pos1, u32 pos2) { + void count2Inc(u32 pos1, u32 pos2) { count2[pos1][pos2]++; } - u32 count1GetNext(u32 &pos1) { + u32 count1GetNext(u32 &pos1) { return count1[pos1]; } - u32 count2GetNext(u32 pos1, u32 &pos2) { + u32 count2GetNext(u32 pos1, u32 &pos2) { return count2[pos1][pos2]; } void backup1(u8 *buf) { @@ -383,16 +383,16 @@ struct Counters { u8 count2High[FSST_CODE_MAX][FSST_CODE_MAX/2]; // array to count subsequent combinations of two symbols in the sample (12-bits: 8-bits low, 4-bits high) u8 count2Low[FSST_CODE_MAX][FSST_CODE_MAX]; // its value is (count2High*256+count2Low) -- but high is 4-bits (we put two numbers in one, hence /2) // 385KB -- but hot area likely just 10 + 30*4 = 130 cache lines (=8KB) - - void count1Set(u32 pos1, u16 val) { + + void count1Set(u32 pos1, u16 val) { count1Low[pos1] = val&255; count1High[pos1] = val>>8; } - void count1Inc(u32 pos1) { + void count1Inc(u32 pos1) { if (!count1Low[pos1]++) // increment high early (when low==0, not when low==255). This means (high > 0) <=> (cnt > 0) count1High[pos1]++; //(0,0)->(1,1)->..->(255,1)->(0,1)->(1,2)->(2,2)->(3,2)..(255,2)->(0,2)->(1,3)->(2,3)... } - void count2Inc(u32 pos1, u32 pos2) { + void count2Inc(u32 pos1, u32 pos2) { if (!count2Low[pos1][pos2]++) // increment high early (when low==0, not when low==255). This means (high > 0) <=> (cnt > 0) // inc 4-bits high counter with 1<<0 (1) or 1<<4 (16) -- depending on whether pos2 is even or odd, repectively count2High[pos1][(pos2)>>1] += 1 << (((pos2)&1)<<2); // we take our chances with overflow.. (4K maxval, on a 8K sample) @@ -432,32 +432,32 @@ struct Counters { memcpy(count1High, buf, FSST_CODE_MAX); memcpy(count1Low, buf+FSST_CODE_MAX, FSST_CODE_MAX); } -}; +}; #endif #define FSST_BUFSZ (3<<19) // 768KB -// an encoder is a symbolmap plus some bufferspace, needed during map construction as well as compression +// an encoder is a symbolmap plus some bufferspace, needed during map construction as well as compression struct Encoder { shared_ptr symbolTable; // symbols, plus metadata and data structures for quick compression (shortCode,hashTab, etc) union { Counters counters; // for counting symbol occurences during map construction - u8 simdbuf[FSST_BUFSZ]; // for compression: SIMD string staging area 768KB = 256KB in + 512KB out (worst case for 256KB in) + u8 simdbuf[FSST_BUFSZ]; // for compression: SIMD string staging area 768KB = 256KB in + 512KB out (worst case for 256KB in) }; }; // job control integer representable in one 64bits SIMD lane: cur/end=input, out=output, pos=which string (2^9=512 per call) struct SIMDjob { - u64 out:19,pos:9,end:18,cur:18; // cur/end is input offsets (2^18=256KB), out is output offset (2^19=512KB) + u64 out:19,pos:9,end:18,cur:18; // cur/end is input offsets (2^18=256KB), out is output offset (2^19=512KB) }; -extern bool +extern bool fsst_hasAVX512(); // runtime check for avx512 capability -extern size_t +extern size_t fsst_compressAVX512( - SymbolTable &symbolTable, + SymbolTable &symbolTable, u8* codeBase, // IN: base address for codes, i.e. compression output (points to simdbuf+256KB) u8* symbolBase, // IN: base address for string bytes, i.e. compression input (points to simdbuf) SIMDjob* input, // IN: input array (size n) with job information: what to encode, where to store it. @@ -465,6 +465,9 @@ fsst_compressAVX512( size_t n, // IN: size of arrays input and output (should be max 512) size_t unroll); // IN: degree of SIMD unrolling +// Symbol manipulation +Symbol concat(Symbol a, Symbol b); + // C++ fsst-compress function with some more control of how the compression happens (algorithm flavor, simd unroll degree) size_t compressImpl(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd); size_t compressAuto(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], int simd); From 12eef2e8d9884ed0c4efff97c91c2b604494a360 Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 24 Nov 2025 08:10:50 +0000 Subject: [PATCH 03/24] cmake build --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 35 + cpp/src/parquet/CMakeLists.txt | 29 +- cpp/src/parquet/decoder.cc | 2 +- cpp/src/parquet/encoder.cc | 2 +- cpp/src/parquet/encoding_test.cc | 2 +- cpp/src/parquet/thirdparty/fsst/fsst.cpp | 200 ------ cpp/src/parquet/thirdparty/fsst/fsst.h | 227 ------ .../parquet/thirdparty/fsst/fsst_avx512.cpp | 149 ---- .../parquet/thirdparty/fsst/fsst_avx512.inc | 57 -- .../thirdparty/fsst/fsst_avx512_unroll1.inc | 57 -- .../thirdparty/fsst/fsst_avx512_unroll2.inc | 114 --- .../thirdparty/fsst/fsst_avx512_unroll3.inc | 171 ----- .../thirdparty/fsst/fsst_avx512_unroll4.inc | 228 ------ cpp/src/parquet/thirdparty/fsst/libfsst.cpp | 651 ------------------ cpp/src/parquet/thirdparty/fsst/libfsst.hpp | 474 ------------- cpp/thirdparty/versions.txt | 3 + 16 files changed, 66 insertions(+), 2335 deletions(-) delete mode 100644 cpp/src/parquet/thirdparty/fsst/fsst.cpp delete mode 100644 cpp/src/parquet/thirdparty/fsst/fsst.h delete mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512.cpp delete mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512.inc delete mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll1.inc delete mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll2.inc delete mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll3.inc delete mode 100644 cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll4.inc delete mode 100644 cpp/src/parquet/thirdparty/fsst/libfsst.cpp delete mode 100644 cpp/src/parquet/thirdparty/fsst/libfsst.hpp diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 4ced2a66bf58..8d67849ea478 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -49,6 +49,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES Boost Brotli BZip2 + fsst c-ares gflags glog @@ -183,6 +184,8 @@ macro(build_dependency DEPENDENCY_NAME) build_brotli() elseif("${DEPENDENCY_NAME}" STREQUAL "BZip2") build_bzip2() + elseif("${DEPENDENCY_NAME}" STREQUAL "fsst") + build_fsst() elseif("${DEPENDENCY_NAME}" STREQUAL "c-ares") build_cares() elseif("${DEPENDENCY_NAME}" STREQUAL "gflags") @@ -382,6 +385,7 @@ endif() if(ARROW_PARQUET) set(ARROW_WITH_RAPIDJSON ON) set(ARROW_WITH_THRIFT ON) + set(ARROW_WITH_FSST ON) endif() if(ARROW_WITH_THRIFT) @@ -637,6 +641,14 @@ else() ) endif() +if(DEFINED ENV{ARROW_FSST_URL}) + set(FSST_SOURCE_URL "$ENV{ARROW_FSST_URL}") +else() + set_urls(FSST_SOURCE_URL + "https://github.com/cwida/fsst/archive/${ARROW_FSST_BUILD_VERSION}.tar.gz" + "${THIRDPARTY_MIRROR_URL}/fsst-${ARROW_FSST_BUILD_VERSION}.tar.gz") +endif() + if(DEFINED ENV{ARROW_GBENCHMARK_URL}) set(GBENCHMARK_SOURCE_URL "$ENV{ARROW_GBENCHMARK_URL}") else() @@ -2604,6 +2616,29 @@ if(ARROW_USE_XSIMD) endif() endif() +function(build_fsst) + message(STATUS "Building FSST from source using FetchContent") + + fetchcontent_declare(fsst + URL ${FSST_SOURCE_URL} + URL_HASH "SHA256=${ARROW_FSST_BUILD_SHA256_CHECKSUM}") + + prepare_fetchcontent() + fetchcontent_makeavailable(fsst) + + set(ARROW_FSST_INCLUDE_DIR + "${fsst_SOURCE_DIR}" + CACHE INTERNAL "FSST include directory") + set(ARROW_FSST_SOURCES + "${fsst_SOURCE_DIR}/libfsst.cpp;${fsst_SOURCE_DIR}/fsst_avx512.cpp" + CACHE INTERNAL "FSST source files") + set(FSST_VENDORED TRUE CACHE INTERNAL "Whether FSST is built from source") +endfunction() + +if(ARROW_WITH_FSST) + resolve_dependency(fsst IS_RUNTIME_DEPENDENCY FALSE) +endif() + macro(build_zlib) message(STATUS "Building ZLIB from source") diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index e06d13d1884e..a2d300d684da 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -42,6 +42,10 @@ function(ADD_PARQUET_TEST REL_TEST_NAME) ${ARGN}) set(TEST_ARGUMENTS PREFIX "parquet" LABELS "parquet-tests") + set(_PARQUET_TEST_EXTRA_ARGS ${ARG_UNPARSED_ARGUMENTS}) + if(PARQUET_TEST_EXTRA_INCLUDES) + list(APPEND _PARQUET_TEST_EXTRA_ARGS EXTRA_INCLUDES ${PARQUET_TEST_EXTRA_INCLUDES}) + endif() if(ARROW_TEST_LINKAGE STREQUAL "static") add_test_case(${REL_TEST_NAME} @@ -49,14 +53,14 @@ function(ADD_PARQUET_TEST REL_TEST_NAME) parquet_static ${PARQUET_TEST_LINK_LIBS} ${TEST_ARGUMENTS} - ${ARG_UNPARSED_ARGUMENTS}) + ${_PARQUET_TEST_EXTRA_ARGS}) else() add_test_case(${REL_TEST_NAME} STATIC_LINK_LIBS parquet_shared ${PARQUET_TEST_LINK_LIBS} ${TEST_ARGUMENTS} - ${ARG_UNPARSED_ARGUMENTS}) + ${_PARQUET_TEST_EXTRA_ARGS}) endif() endfunction() @@ -134,6 +138,9 @@ elseif(NOT MSVC) list(APPEND PARQUET_TEST_LINK_LIBS ${CMAKE_DL_LIBS}) endif() +set(PARQUET_TEST_EXTRA_INCLUDES) +set(PARQUET_PRIVATE_INCLUDE_DIRS) + # # Generated Thrift sources set(PARQUET_THRIFT_SOURCE_DIR "${ARROW_SOURCE_DIR}/src/generated/") @@ -171,8 +178,6 @@ set(PARQUET_SRCS encryption/internal_file_encryptor.cc exception.cc file_reader.cc - thirdparty/fsst/libfsst.cpp - thirdparty/fsst/fsst_avx512.cpp file_writer.cc geospatial/statistics.cc geospatial/util_internal.cc @@ -193,6 +198,14 @@ set(PARQUET_SRCS stream_writer.cc types.cc) +if(DEFINED ARROW_FSST_SOURCES) + list(APPEND PARQUET_SRCS ${ARROW_FSST_SOURCES}) +endif() +if(DEFINED ARROW_FSST_INCLUDE_DIR) + list(APPEND PARQUET_PRIVATE_INCLUDE_DIRS ${ARROW_FSST_INCLUDE_DIR}) + list(APPEND PARQUET_TEST_EXTRA_INCLUDES ${ARROW_FSST_INCLUDE_DIR}) +endif() + if(ARROW_HAVE_RUNTIME_AVX2) # AVX2 is used as a proxy for BMI2. list(APPEND PARQUET_SRCS level_comparison_avx2.cc level_conversion_bmi2.cc) @@ -308,6 +321,14 @@ add_arrow_lib(parquet STATIC_INSTALL_INTERFACE_LIBS ${PARQUET_STATIC_INSTALL_INTERFACE_LIBS}) +if(PARQUET_PRIVATE_INCLUDE_DIRS) + foreach(_parquet_target parquet_objlib parquet_shared parquet_static) + if(TARGET ${_parquet_target}) + target_include_directories(${_parquet_target} PRIVATE ${PARQUET_PRIVATE_INCLUDE_DIRS}) + endif() + endforeach() +endif() + if(WIN32 AND NOT (ARROW_TEST_LINKAGE STREQUAL "static")) add_library(parquet_test_support STATIC "${PARQUET_THRIFT_SOURCE_DIR}/parquet_types.cpp") diff --git a/cpp/src/parquet/decoder.cc b/cpp/src/parquet/decoder.cc index 77bc3c760ff2..42838fd059f1 100644 --- a/cpp/src/parquet/decoder.cc +++ b/cpp/src/parquet/decoder.cc @@ -53,7 +53,7 @@ #include "parquet/exception.h" #include "parquet/platform.h" #include "parquet/schema.h" -#include "parquet/thirdparty/fsst/fsst.h" +#include "fsst.h" #include "parquet/types.h" #ifdef _MSC_VER diff --git a/cpp/src/parquet/encoder.cc b/cpp/src/parquet/encoder.cc index 61b1bb9606c3..00daa366f6ab 100644 --- a/cpp/src/parquet/encoder.cc +++ b/cpp/src/parquet/encoder.cc @@ -50,7 +50,7 @@ #include "parquet/exception.h" #include "parquet/platform.h" #include "parquet/schema.h" -#include "parquet/thirdparty/fsst/fsst.h" +#include "fsst.h" #include "parquet/types.h" #ifdef _MSC_VER diff --git a/cpp/src/parquet/encoding_test.cc b/cpp/src/parquet/encoding_test.cc index ca51d8e1b241..a43bafd0960b 100644 --- a/cpp/src/parquet/encoding_test.cc +++ b/cpp/src/parquet/encoding_test.cc @@ -53,7 +53,7 @@ #include "parquet/platform.h" #include "parquet/schema.h" #include "parquet/test_util.h" -#include "parquet/thirdparty/fsst/fsst.h" +#include "fsst.h" #include "parquet/types.h" using arrow::default_memory_pool; diff --git a/cpp/src/parquet/thirdparty/fsst/fsst.cpp b/cpp/src/parquet/thirdparty/fsst/fsst.cpp deleted file mode 100644 index c1a9cc349803..000000000000 --- a/cpp/src/parquet/thirdparty/fsst/fsst.cpp +++ /dev/null @@ -1,200 +0,0 @@ -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// -// Copyright 2018-2019, CWI, TU Munich, FSU Jena -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -#ifdef FSST12 -#include "fsst12.h" // the official FSST API -- also usable by C mortals -#else -#include "fsst.h" // the official FSST API -- also usable by C mortals -#endif -#include -#include -#include -#include -#include -#include -using namespace std; - -// Utility to compress and decompress (-d) data with FSST (using stdin and stdout). -// -// The utility has a poor-man's async I/O in that it uses double buffering for input and output, -// and two background pthreads for reading and writing. The idea is to make the CPU overlap with I/O. -// -// The data format is quite simple. A FSST compressed file is a sequence of blocks, each with format: -// (1) 3-byte block length field (max blocksize is hence 16MB). This byte-length includes (1), (2) and (3). -// (2) FSST dictionary as produced by fst_export(). -// (3) the FSST compressed data. -// -// The natural strength of FSST is in fact not block-based compression, but rather the compression and -// *individual* decompression of many small strings separately. Think of compressed databases and (column-store) -// data formats. But, this utility is to serve as an apples-to-apples comparison point with utilities like lz4. - -namespace { - -class BinarySemaphore { - private: - mutex m; - condition_variable cv; - bool value; - - public: - explicit BinarySemaphore(bool initialValue = false) : value(initialValue) {} - void wait() { - unique_lock lock(m); - while (!value) cv.wait(lock); - value = false; - } - void post() { - { unique_lock lock(m); value = true; } - cv.notify_one(); - } -}; - -bool stopThreads = false; -BinarySemaphore srcDoneIO[2], dstDoneIO[2], srcDoneCPU[2], dstDoneCPU[2]; -unsigned char *srcBuf[2] = { NULL, NULL }; -unsigned char *dstBuf[2] = { NULL, NULL }; -unsigned char *dstMem[2] = { NULL, NULL }; -size_t srcLen[2] = { 0, 0 }; -size_t dstLen[2] = { 0, 0 }; - -#define FSST_MEMBUF (1ULL<<22) -int decompress = 0; -size_t blksz = FSST_MEMBUF-(1+FSST_MAXHEADER/2); // block size of compression (max compressed size must fit 3 bytes) - -#define DESERIALIZE(p) (((unsigned long long) (p)[0]) << 16) | (((unsigned long long) (p)[1]) << 8) | ((unsigned long long) (p)[2]) -#define SERIALIZE(l,p) { (p)[0] = ((l)>>16)&255; (p)[1] = ((l)>>8)&255; (p)[2] = (l)&255; } - -void reader(ifstream& src) { - for(int swap=0; true; swap = 1-swap) { - srcDoneCPU[swap].wait(); - if (stopThreads) break; - src.read((char*) srcBuf[swap], blksz); - srcLen[swap] = (unsigned long) src.gcount(); - if (decompress) { - if (blksz && srcLen[swap] == blksz) { - blksz = DESERIALIZE(srcBuf[swap]+blksz-3); // read size of next block - srcLen[swap] -= 3; // cut off size bytes - } else { - blksz = 0; - } - } - srcDoneIO[swap].post(); - } -} - -void writer(ofstream& dst) { - for(int swap=0; true; swap = 1-swap) { - dstDoneCPU[swap].wait(); - if (!dstLen[swap]) break; - dst.write((char*) dstBuf[swap], dstLen[swap]); - dstDoneIO[swap].post(); - } - for(int swap=0; swap<2; swap++) - dstDoneIO[swap].post(); -} - -} - -#ifdef FSST_STANDALONE -int main(int argc, char* argv[]) { - size_t srcTot = 0, dstTot = 0; - if (argc < 2 || argc > 4 || (argc == 4 && (argv[1][0] != '-' || argv[1][1] != 'd' || argv[1][2]))) { - cerr << "usage: " << argv[0] << " -d infile outfile" << endl; - cerr << " " << argv[0] << " infile outfile" << endl; - cerr << " " << argv[0] << " infile" << endl; - return -1; - } - decompress = (argc == 4); - string srcfile(argv[1+decompress]), dstfile; - if (argc == 2) { - dstfile = srcfile + ".fsst"; - } else { - dstfile = argv[2+decompress]; - } - ifstream src; - ofstream dst; - src.open(srcfile, ios::binary); - dst.open(dstfile, ios::binary); - dst.exceptions(ios_base::failbit); - dst.exceptions(ios_base::badbit); - src.exceptions(ios_base::badbit); - if (decompress) { - unsigned char tmp[3]; - src.read((char*) tmp, 3); - if (src.gcount() != 3) { - cerr << "failed to open input." << endl; - return -1; - } - blksz = DESERIALIZE(tmp); // read first block size - } - vector buffer(FSST_MEMBUF*6); - srcBuf[0] = buffer.data(); - srcBuf[1] = srcBuf[0] + (FSST_MEMBUF*(1ULL+decompress)); - dstMem[0] = srcBuf[1] + (FSST_MEMBUF*(1ULL+decompress)); - dstMem[1] = dstMem[0] + (FSST_MEMBUF*(2ULL-decompress)); - - for(int swap=0; swap<2; swap++) { - srcDoneCPU[swap].post(); // input buffer is not being processed initially - dstDoneIO[swap].post(); // output buffer is not being written initially - } - thread readerThread([&src]{ reader(src); }); - thread writerThread([&dst]{ writer(dst); }); - - for(int swap=0; true; swap = 1-swap) { - srcDoneIO[swap].wait(); // wait until input buffer is available (i.e. done reading) - dstDoneIO[swap].wait(); // wait until output buffer is ready writing hence free for use - if (srcLen[swap] == 0) { - dstLen[swap] = 0; - break; - } - if (decompress) { - fsst_decoder_t decoder; - size_t hdr = fsst_import(&decoder, srcBuf[swap]); - dstLen[swap] = fsst_decompress(&decoder, srcLen[swap] - hdr, srcBuf[swap] + hdr, FSST_MEMBUF, dstBuf[swap] = dstMem[swap]); - } else { - unsigned char tmp[FSST_MAXHEADER]; - fsst_encoder_t* encoder = fsst_create(1, &srcLen[swap], const_cast(&srcBuf[swap]), 0); - size_t hdr = fsst_export(encoder, tmp); - if (fsst_compress(encoder, 1, &srcLen[swap], const_cast(&srcBuf[swap]), - FSST_MEMBUF * 2, dstMem[swap] + FSST_MAXHEADER + 3, - &dstLen[swap], &dstBuf[swap]) < 1) - return -1; - dstLen[swap] += 3 + hdr; - dstBuf[swap] -= 3 + hdr; - SERIALIZE(dstLen[swap],dstBuf[swap]); // block starts with size - copy(tmp, tmp+hdr, dstBuf[swap]+3); // then the header (followed by the compressed bytes which are already there) - fsst_destroy(encoder); - } - srcTot += srcLen[swap]; - dstTot += dstLen[swap]; - srcDoneCPU[swap].post(); // input buffer may be re-used by the reader for the next block - dstDoneCPU[swap].post(); // output buffer is ready for writing out - } - cerr << (decompress?"Dec":"C") << "ompressed " << srcTot << " bytes into " << dstTot << " bytes ==> " << (int) ((100*dstTot)/srcTot) << "%" << endl; - - // force wait until all background writes finished - stopThreads = true; - for(int swap=0; swap<2; swap++) { - srcDoneCPU[swap].post(); - dstDoneCPU[swap].post(); - } - dstDoneIO[0].wait(); - dstDoneIO[1].wait(); - readerThread.join(); - writerThread.join(); -} -#endif // FSST_STANDALONE diff --git a/cpp/src/parquet/thirdparty/fsst/fsst.h b/cpp/src/parquet/thirdparty/fsst/fsst.h deleted file mode 100644 index 71085d57201d..000000000000 --- a/cpp/src/parquet/thirdparty/fsst/fsst.h +++ /dev/null @@ -1,227 +0,0 @@ -/* - * the API for FSST compression -- (c) Peter Boncz, Viktor Leis and Thomas Neumann (CWI, TU Munich), 2018-2019 - * - * =================================================================================================================================== - * this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): - * - * Copyright 2018-2020, CWI, TU Munich, FSU Jena - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files - * (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, - * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR - * IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - * You can contact the authors via the FSST source repository : https://github.com/cwida/fsst - * =================================================================================================================================== - * - * FSST: Fast Static Symbol Table compression - * see the paper https://github.com/cwida/fsst/raw/master/fsstcompression.pdf - * - * FSST is a compression scheme focused on string/text data: it can compress strings from distributions with many different values (i.e. - * where dictionary compression will not work well). It allows *random-access* to compressed data: it is not block-based, so individual - * strings can be decompressed without touching the surrounding data in a compressed block. When compared to e.g. lz4 (which is - * block-based), FSST achieves similar decompression speed, (2x) better compression speed and 30% better compression ratio on text. - * - * FSST encodes strings also using a symbol table -- but it works on pieces of the string, as it maps "symbols" (1-8 byte sequences) - * onto "codes" (single-bytes). FSST can also represent a byte as an exception (255 followed by the original byte). Hence, compression - * transforms a sequence of bytes into a (supposedly shorter) sequence of codes or escaped bytes. These shorter byte-sequences could - * be seen as strings again and fit in whatever your program is that manipulates strings. - * - * useful property: FSST ensures that strings that are equal, are also equal in their compressed form. - * - * In this API, strings are considered byte-arrays (byte = unsigned char) and a batch of strings is represented as an array of - * unsigned char* pointers to their starts. A seperate length array (of unsigned int) denotes how many bytes each string consists of. - * - * This representation as unsigned char* pointers tries to assume as little as possible on the memory management of the program - * that calls this API, and is also intended to allow passing strings into this API without copying (even if you use C++ strings). - * - * We optionally support C-style zero-terminated strings (zero appearing only at the end). In this case, the compressed strings are - * also zero-terminated strings. In zero-terminated mode, the zero-byte at the end *is* counted in the string byte-length. - */ -#ifndef FSST_INCLUDED_H -#define FSST_INCLUDED_H - -#ifdef _MSC_VER -#define __restrict__ -#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#define __ORDER_LITTLE_ENDIAN__ 2 -#include -static inline int __builtin_ctzl(unsigned long long x) { - unsigned long ret; - _BitScanForward64(&ret, x); - return (int)ret; -} -#endif - -#ifdef __cplusplus -#define FSST_FALLTHROUGH [[fallthrough]] -#include -extern "C" { -#else -#define FSST_FALLTHROUGH -#endif - -#include - -/* A compressed string is simply a string of 1-byte codes; except for code 255, which is followed by an uncompressed byte. */ -#define FSST_ESC 255 - -/* Data structure needed for compressing strings - use fsst_duplicate() to create thread-local copies. Use fsst_destroy() to free. */ -typedef void* fsst_encoder_t; /* opaque type - it wraps around a rather large (~900KB) C++ object */ - -/* Data structure needed for decompressing strings - read-only and thus can be shared between multiple decompressing threads. */ -typedef struct { - unsigned long long version; /* version id */ - unsigned char zeroTerminated; /* terminator is a single-byte code that does not appear in longer symbols */ - unsigned char len[255]; /* len[x] is the byte-length of the symbol x (1 < len[x] <= 8). */ - unsigned long long symbol[255]; /* symbol[x] contains in LITTLE_ENDIAN the bytesequence that code x represents (0 <= x < 255). */ -} fsst_decoder_t; - -/* Calibrate a FSST symboltable from a batch of strings (it is best to provide at least 16KB of data). */ -fsst_encoder_t* -fsst_create( - size_t n, /* IN: number of strings in batch to sample from. */ - const size_t lenIn[], /* IN: byte-lengths of the inputs */ - const unsigned char *strIn[], /* IN: string start pointers. */ - int zeroTerminated /* IN: whether input strings are zero-terminated. If so, encoded strings are as well (i.e. symbol[0]=""). */ -); - -/* Create another encoder instance, necessary to do multi-threaded encoding using the same symbol table. */ -fsst_encoder_t* -fsst_duplicate( - fsst_encoder_t *encoder /* IN: the symbol table to duplicate. */ -); - -#define FSST_MAXHEADER (8+1+8+2048+1) /* maxlen of deserialized fsst header, produced/consumed by fsst_export() resp. fsst_import() */ - -/* Space-efficient symbol table serialization (smaller than sizeof(fsst_decoder_t) - by saving on the unused bytes in symbols of len < 8). */ -unsigned int /* OUT: number of bytes written in buf, at most sizeof(fsst_decoder_t) */ -fsst_export( - fsst_encoder_t *encoder, /* IN: the symbol table to dump. */ - unsigned char *buf /* OUT: pointer to a byte-buffer where to serialize this symbol table. */ -); - -/* Deallocate encoder. */ -void -fsst_destroy(fsst_encoder_t*); - -/* Return a decoder structure from serialized format (typically used in a block-, file- or row-group header). */ -unsigned int /* OUT: number of bytes consumed in buf (0 on failure). */ -fsst_import( - fsst_decoder_t *decoder, /* IN: this symbol table will be overwritten. */ - unsigned char const *buf /* IN: pointer to a byte-buffer where fsst_export() serialized this symbol table. */ -); - -/* Return a decoder structure from an encoder. */ -fsst_decoder_t -fsst_decoder( - fsst_encoder_t *encoder -); - -/* Compress a batch of strings (on AVX512 machines best performance is obtained by compressing more than 32KB of string volume). */ -/* The output buffer must be large; at least "conservative space" (7+2*inputlength) for the first string for something to happen. */ -size_t /* OUT: the number of compressed strings (<=n) that fit the output buffer. */ -fsst_compress( - fsst_encoder_t *encoder, /* IN: encoder obtained from fsst_create(). */ - size_t nstrings, /* IN: number of strings in batch to compress. */ - const size_t lenIn[], /* IN: byte-lengths of the inputs */ - const unsigned char *strIn[], /* IN: input string start pointers. */ - size_t outsize, /* IN: byte-length of output buffer. */ - unsigned char *output, /* OUT: memory buffer to put the compressed strings in (one after the other). */ - size_t lenOut[], /* OUT: byte-lengths of the compressed strings. */ - unsigned char *strOut[] /* OUT: output string start pointers. Will all point into [output,output+size). */ -); - -/* Decompress a single string, inlined for speed. */ -inline size_t /* OUT: bytesize of the decompressed string. If > size, the decoded output is truncated to size. */ -fsst_decompress( - const fsst_decoder_t *decoder, /* IN: use this symbol table for compression. */ - size_t lenIn, /* IN: byte-length of compressed string. */ - const unsigned char *strIn, /* IN: compressed string. */ - size_t size, /* IN: byte-length of output buffer. */ - unsigned char *output /* OUT: memory buffer to put the decompressed string in. */ -) { - unsigned char*__restrict__ len = (unsigned char* __restrict__) decoder->len; - unsigned char*__restrict__ strOut = (unsigned char* __restrict__) output; - unsigned long long*__restrict__ symbol = (unsigned long long* __restrict__) decoder->symbol; - size_t code, posOut = 0, posIn = 0; -#ifndef FSST_MUST_ALIGN /* defining on platforms that require aligned memory access may help their performance */ -#define FSST_UNALIGNED_STORE(dst,src) memcpy((unsigned long long*) (dst), &(src), sizeof(unsigned long long)) -#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) - while (posOut+32 <= size && posIn+4 <= lenIn) { - unsigned int nextBlock, escapeMask; - memcpy(&nextBlock, strIn+posIn, sizeof(unsigned int)); - escapeMask = (nextBlock&0x80808080u)&((((~nextBlock)&0x7F7F7F7Fu)+0x7F7F7F7Fu)^0x80808080u); - if (escapeMask == 0) { - code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; - code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; - code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; - code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; - } else { - unsigned long firstEscapePos=__builtin_ctzl((unsigned long long) escapeMask)>>3; - switch(firstEscapePos) { /* Duff's device */ - case 3: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; - // fall through - case 2: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; - // fall through - case 1: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; - // fall through - case 0: posIn+=2; strOut[posOut++] = strIn[posIn-1]; /* decompress an escaped byte */ - } - } - } - if (posOut+32 <= size) { // handle the possibly 3 last bytes without a loop - if (posIn+2 <= lenIn) { - strOut[posOut] = strIn[posIn+1]; - if (strIn[posIn] != FSST_ESC) { - code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; - if (strIn[posIn] != FSST_ESC) { - code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; - } else { - posIn += 2; strOut[posOut++] = strIn[posIn-1]; - } - } else { - posIn += 2; posOut++; - } - } - if (posIn < lenIn) { // last code cannot be an escape - code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; - } - } -#else - while (posOut+8 <= size && posIn < lenIn) - if ((code = strIn[posIn++]) < FSST_ESC) { /* symbol compressed as code? */ - FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); /* unaligned memory write */ - posOut += len[code]; - } else { - strOut[posOut] = strIn[posIn]; /* decompress an escaped byte */ - posIn++; posOut++; - } -#endif -#endif - while (posIn < lenIn) - if ((code = strIn[posIn++]) < FSST_ESC) { - size_t posWrite = posOut, endWrite = posOut + len[code]; - unsigned char* __restrict__ symbolPointer = ((unsigned char* __restrict__) &symbol[code]) - posWrite; - if ((posOut = endWrite) > size) endWrite = size; - for(; posWrite < endWrite; posWrite++) /* only write if there is room */ - strOut[posWrite] = symbolPointer[posWrite]; - } else { - if (posOut < size) strOut[posOut] = strIn[posIn]; /* idem */ - posIn++; posOut++; - } - if (posOut >= size && (decoder->zeroTerminated&1)) strOut[size-1] = 0; - return posOut; /* full size of decompressed string (could be >size, then the actually decompressed part) */ -} - -#ifdef __cplusplus -} -#endif -#endif /* FSST_INCLUDED_H */ diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512.cpp b/cpp/src/parquet/thirdparty/fsst/fsst_avx512.cpp deleted file mode 100644 index e43d3e03652c..000000000000 --- a/cpp/src/parquet/thirdparty/fsst/fsst_avx512.cpp +++ /dev/null @@ -1,149 +0,0 @@ -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -#include "libfsst.hpp" - -#if defined(__x86_64__) || defined(_M_X64) -#include - -#ifdef _WIN32 -namespace libfsst { -bool fsst_hasAVX512() { - int info[4]; - __cpuidex(info, 0x00000007, 0); - return (info[1]>>16)&1; -} -} // namespace libfsst -#else -#include -namespace libfsst { -bool fsst_hasAVX512() { - int info[4]; - __cpuid_count(0x00000007, 0, info[0], info[1], info[2], info[3]); - return (info[1]>>16)&1; -} -} // namespace libfsst -#endif -#else -namespace libfsst { -bool fsst_hasAVX512() { return false; } -} // namespace libfsst -#endif - -namespace libfsst { - -// BULK COMPRESSION OF STRINGS -// -// In one call of this function, we can compress 512 strings, each of maximum length 511 bytes. -// strings can be shorter than 511 bytes, no problem, but if they are longer we need to cut them up. -// -// In each iteration of the while loop, we find one code in each of the unroll*8 strings, i.e. (8,16,24 or 32) for resp. unroll=1,2,3,4 -// unroll3 performs best on my hardware -// -// In the worst case, each final encoded string occupies 512KB bytes (512*1024; with 1024=512xexception, exception = 2 bytes). -// - hence codeBase is a buffer of 512KB (needs 19 bits jobs), symbolBase of 256KB (needs 18 bits jobs). -// -// 'jobX' controls the encoding of each string and is therefore a u64 with format [out:19][pos:9][end:18][cur:18] (low-to-high bits) -// The field 'pos' tells which string we are processing (0..511). We need this info as strings will complete compressing out-of-order. -// -// Strings will have different lengths, and when a string is finished, we reload from the buffer of 512 input strings. -// This continues until we have less than (8,16,24 or 32; depending on unroll) strings left to process. -// - so 'processed' is the amount of strings we started processing and it is between [480,512]. -// Note that when we quit, there will still be some (<32) strings that we started to process but which are unfinished. -// - so 'unfinished' is that amount. These unfinished strings will be encoded further using the scalar method. -// -// Apart from the coded strings, we return in a output[] array of size 'processed' the job values of the 'finished' strings. -// In the following 'unfinished' slots (processed=finished+unfinished) we output the 'job' values of the unfinished strings. -// -// For the finished strings, we need [out:19] to see the compressed size and [pos:9] to see which string we refer to. -// For the unfinished strings, we need all fields of 'job' to continue the compression with scalar code (see SIMD code in compressBatch). -// -// THIS IS A SEPARATE CODE FILE NOT BECAUSE OF MY LOVE FOR MODULARIZED CODE BUT BECAUSE IT ALLOWS TO COMPILE IT WITH DIFFERENT FLAGS -// in particular, unrolling is crucial for gather/scatter performance, but requires registers. the #define all_* expressions however, -// will be detected to be constants by g++ -O2 and will be precomputed and placed into AVX512 registers - spoiling 9 of them. -// This reduces the effectiveness of unrolling, hence -O2 makes the loop perform worse than -O1 which skips this optimization. -// Assembly inspection confirmed that 3-way unroll with -O1 avoids needless load/stores. - -size_t fsst_compressAVX512(SymbolTable &symbolTable, u8* codeBase, u8* symbolBase, SIMDjob *input, SIMDjob *output, size_t n, size_t unroll) { - size_t processed = 0; - // define some constants (all_x means that all 8 lanes contain 64-bits value X) -#ifdef __AVX512F__ - //__m512i all_suffixLim= _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) symbolTable->suffixLim)); -- for variants b,c - __m512i all_MASK = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) -1)); - __m512i all_PRIME = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) FSST_HASH_PRIME)); - __m512i all_ICL_FREE = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) FSST_ICL_FREE)); -#define all_HASH _mm512_srli_epi64(all_MASK, 64-FSST_HASH_LOG2SIZE) -#define all_ONE _mm512_srli_epi64(all_MASK, 63) -#define all_M19 _mm512_srli_epi64(all_MASK, 45) -#define all_M18 _mm512_srli_epi64(all_MASK, 46) -#define all_M28 _mm512_srli_epi64(all_MASK, 36) -#define all_FFFFFF _mm512_srli_epi64(all_MASK, 40) -#define all_FFFF _mm512_srli_epi64(all_MASK, 48) -#define all_FF _mm512_srli_epi64(all_MASK, 56) - - SIMDjob *inputEnd = input+n; - assert(n >= unroll*8 && n <= 512); // should be close to 512 - __m512i job1, job2, job3, job4; // will contain current jobs, for each unroll 1,2,3,4 - __mmask8 loadmask1 = 255, loadmask2 = 255*(unroll>1), loadmask3 = 255*(unroll>2), loadmask4 = 255*(unroll>3); // 2b loaded new strings bitmask per unroll - u32 delta1 = 8, delta2 = 8*(unroll>1), delta3 = 8*(unroll>2), delta4 = 8*(unroll>3); // #new loads this SIMD iteration per unroll - - if (unroll >= 4) { - while (input+delta1+delta2+delta3+delta4 < inputEnd) { - #include "fsst_avx512_unroll4.inc" - } - } else if (unroll == 3) { - while (input+delta1+delta2+delta3 < inputEnd) { - #include "fsst_avx512_unroll3.inc" - } - } else if (unroll == 2) { - while (input+delta1+delta2 < inputEnd) { - #include "fsst_avx512_unroll2.inc" - } - } else { - while (input+delta1 < inputEnd) { - #include "fsst_avx512_unroll1.inc" - } - } - - // flush the job states of the unfinished strings at the end of output[] - processed = n - (inputEnd - input); - u32 unfinished = 0; - if (unroll > 1) { - if (unroll > 2) { - if (unroll > 3) { - _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask4=~loadmask4, job4); - unfinished += _mm_popcnt_u32((int) loadmask4); - } - _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask3=~loadmask3, job3); - unfinished += _mm_popcnt_u32((int) loadmask3); - } - _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask2=~loadmask2, job2); - unfinished += _mm_popcnt_u32((int) loadmask2); - } - _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask1=~loadmask1, job1); -#else - (void) symbolTable; - (void) codeBase; - (void) symbolBase; - (void) input; - (void) output; - (void) n; - (void) unroll; -#endif - return processed; -} -} // namespace libfsst diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512.inc b/cpp/src/parquet/thirdparty/fsst/fsst_avx512.inc deleted file mode 100644 index 0a74541dd884..000000000000 --- a/cpp/src/parquet/thirdparty/fsst/fsst_avx512.inc +++ /dev/null @@ -1,57 +0,0 @@ -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -// - // load new jobs in the empty lanes (initially, all lanes are empty, so loadmaskX=11111111, deltaX=8). - jobX = _mm512_mask_expandloadu_epi64(jobX, loadmaskX, input); input += deltaX; - // load the next 8 input string bytes (uncompressed data, aka 'symbols'). - __m512i wordX = _mm512_i64gather_epi64(_mm512_srli_epi64(jobX, 46), symbolBase, 1); - // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. - // codeX: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). - __m512i codeX = _mm512_i64gather_epi64(_mm512_and_epi64(wordX, all_FFFF), symbolTable.shortCodes, sizeof(u16)); - // get the first three bytes of the string. - __m512i posX = _mm512_mullo_epi64(_mm512_and_epi64(wordX, all_FFFFFF), all_PRIME); - // hash them into a random number: posX = posX*PRIME; posX ^= posX>>SHIFT - posX = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(posX,_mm512_srli_epi64(posX,FSST_SHIFT)), all_HASH), 4); - // lookup in the 3-byte-prefix keyed hash table - __m512i iclX = _mm512_i64gather_epi64(posX, (((char*) symbolTable.hashTab) + 8), 1); - // speculatively store the first input byte into the second position of the writeX register (in case it turns out to be an escaped byte). - __m512i writeX = _mm512_slli_epi64(_mm512_and_epi64(wordX, all_FF), 8); - // lookup just like the iclX above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. - __m512i symbX = _mm512_i64gather_epi64(posX, (((char*) symbolTable.hashTab) + 0), 1); - // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). - posX = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(iclX, all_FF)); - // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). - __mmask8 matchX = _mm512_cmpeq_epi64_mask(symbX, _mm512_and_epi64(wordX, posX)) & _mm512_cmplt_epi64_mask(iclX, all_ICL_FREE); - // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. - codeX = _mm512_mask_mov_epi64(codeX, matchX, _mm512_srli_epi64(iclX, 16)); - // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. - writeX = _mm512_or_epi64(writeX, _mm512_and_epi64(codeX, all_FF)); - // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) - codeX = _mm512_and_epi64(codeX, all_FFFF); - // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) - _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(jobX, all_M19), writeX, 1); - // increase the jobX.cur field in the job with the symbol length (for this, shift away 12 bits from the code) - jobX = _mm512_add_epi64(jobX, _mm512_slli_epi64(_mm512_srli_epi64(codeX, FSST_LEN_BITS), 46)); - // increase the jobX.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) - jobX = _mm512_add_epi64(jobX, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(codeX, 8), all_ONE))); - // test which lanes are done now (jobX.cur==jobX.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the jobX register) - loadmaskX = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(jobX, 46), _mm512_and_epi64(_mm512_srli_epi64(jobX, 28), all_M18)); - // calculate the amount of lanes in jobX that are done - deltaX = _mm_popcnt_u32((int) loadmaskX); - // write out the job state for the lanes that are done (we need the final 'jobX.out' value to compute the compressed string length) - _mm512_mask_compressstoreu_epi64(output, loadmaskX, jobX); output += deltaX; diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll1.inc b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll1.inc deleted file mode 100644 index f4b81c7970dd..000000000000 --- a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll1.inc +++ /dev/null @@ -1,57 +0,0 @@ -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -// - // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). - job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; - // load the next 8 input string bytes (uncompressed data, aka 'symbols'). - __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); - // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. - // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). - __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); - // get the first three bytes of the string. - __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); - // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT - pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); - // lookup in the 3-byte-prefix keyed hash table - __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); - // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). - __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); - // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. - __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); - // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). - pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); - // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). - __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); - // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. - code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); - // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. - write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); - // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) - code1 = _mm512_and_epi64(code1, all_FFFF); - // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) - _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); - // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) - job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); - // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) - job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); - // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) - loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); - // calculate the amount of lanes in job1 that are done - delta1 = _mm_popcnt_u32((int) loadmask1); - // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) - _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll2.inc b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll2.inc deleted file mode 100644 index aa33cd7e69c5..000000000000 --- a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll2.inc +++ /dev/null @@ -1,114 +0,0 @@ -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// -// -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// furnished to do so, subject to the following conditions: -// -// -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// -// -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -// -// - // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). - // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). - job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; - job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; - // load the next 8 input string bytes (uncompressed data, aka 'symbols'). - // load the next 8 input string bytes (uncompressed data, aka 'symbols'). - __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); - __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); - // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. - // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. - // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). - // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). - __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); - __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); - // get the first three bytes of the string. - // get the first three bytes of the string. - __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); - __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); - // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT - // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT - pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); - pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); - // lookup in the 3-byte-prefix keyed hash table - // lookup in the 3-byte-prefix keyed hash table - __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); - __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); - // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). - // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). - __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); - __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); - // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. - // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. - __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); - __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); - // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). - // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). - pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); - pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); - // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). - // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). - __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); - __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); - // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. - // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. - code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); - code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); - // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. - // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. - write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); - write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); - // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) - // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) - code1 = _mm512_and_epi64(code1, all_FFFF); - code2 = _mm512_and_epi64(code2, all_FFFF); - // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) - // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) - _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); - _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); - // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) - // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) - job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); - job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); - // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) - // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) - job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); - job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); - // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) - // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) - loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); - loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); - // calculate the amount of lanes in job1 that are done - // calculate the amount of lanes in job2 that are done - delta1 = _mm_popcnt_u32((int) loadmask1); - delta2 = _mm_popcnt_u32((int) loadmask2); - // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) - // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) - _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; - _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll3.inc b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll3.inc deleted file mode 100644 index e2057032abd3..000000000000 --- a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll3.inc +++ /dev/null @@ -1,171 +0,0 @@ -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// -// -// -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// -// -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// furnished to do so, subject to the following conditions: -// furnished to do so, subject to the following conditions: -// -// -// -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E3PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// -// -// -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -// -// -// - // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). - // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). - // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask3=11111111, delta3=8). - job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; - job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; - job3 = _mm512_mask_expandloadu_epi64(job3, loadmask3, input); input += delta3; - // load the next 8 input string bytes (uncompressed data, aka 'symbols'). - // load the next 8 input string bytes (uncompressed data, aka 'symbols'). - // load the next 8 input string bytes (uncompressed data, aka 'symbols'). - __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); - __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); - __m512i word3 = _mm512_i64gather_epi64(_mm512_srli_epi64(job3, 46), symbolBase, 1); - // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. - // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. - // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. - // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). - // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). - // code3: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). - __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); - __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); - __m512i code3 = _mm512_i64gather_epi64(_mm512_and_epi64(word3, all_FFFF), symbolTable.shortCodes, sizeof(u16)); - // get the first three bytes of the string. - // get the first three bytes of the string. - // get the first three bytes of the string. - __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); - __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); - __m512i pos3 = _mm512_mullo_epi64(_mm512_and_epi64(word3, all_FFFFFF), all_PRIME); - // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT - // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT - // hash them into a random number: pos3 = pos3*PRIME; pos3 ^= pos3>>SHIFT - pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); - pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); - pos3 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos3,_mm512_srli_epi64(pos3,FSST_SHIFT)), all_HASH), 4); - // lookup in the 3-byte-prefix keyed hash table - // lookup in the 3-byte-prefix keyed hash table - // lookup in the 3-byte-prefix keyed hash table - __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); - __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); - __m512i icl3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 8), 1); - // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). - // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). - // speculatively store the first input byte into the second position of the write3 register (in case it turns out to be an escaped byte). - __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); - __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); - __m512i write3 = _mm512_slli_epi64(_mm512_and_epi64(word3, all_FF), 8); - // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. - // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. - // lookup just like the icl3 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. - __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); - __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); - __m512i symb3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 0), 1); - // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). - // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). - // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). - pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); - pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); - pos3 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl3, all_FF)); - // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). - // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). - // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). - __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); - __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); - __mmask8 match3 = _mm512_cmpeq_epi64_mask(symb3, _mm512_and_epi64(word3, pos3)) & _mm512_cmplt_epi64_mask(icl3, all_ICL_FREE); - // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. - // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. - // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. - code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); - code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); - code3 = _mm512_mask_mov_epi64(code3, match3, _mm512_srli_epi64(icl3, 16)); - // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. - // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. - // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. - write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); - write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); - write3 = _mm512_or_epi64(write3, _mm512_and_epi64(code3, all_FF)); - // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) - // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) - // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) - code1 = _mm512_and_epi64(code1, all_FFFF); - code2 = _mm512_and_epi64(code2, all_FFFF); - code3 = _mm512_and_epi64(code3, all_FFFF); - // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) - // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) - // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) - _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); - _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); - _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job3, all_M19), write3, 1); - // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) - // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) - // increase the job3.cur field in the job with the symbol length (for this, shift away 12 bits from the code) - job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); - job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); - job3 = _mm512_add_epi64(job3, _mm512_slli_epi64(_mm512_srli_epi64(code3, FSST_LEN_BITS), 46)); - // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) - // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) - // increase the job3.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) - job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); - job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); - job3 = _mm512_add_epi64(job3, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code3, 8), all_ONE))); - // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) - // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) - // test which lanes are done now (job3.cur==job3.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job3 register) - loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); - loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); - loadmask3 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job3, 46), _mm512_and_epi64(_mm512_srli_epi64(job3, 28), all_M18)); - // calculate the amount of lanes in job1 that are done - // calculate the amount of lanes in job2 that are done - // calculate the amount of lanes in job3 that are done - delta1 = _mm_popcnt_u32((int) loadmask1); - delta2 = _mm_popcnt_u32((int) loadmask2); - delta3 = _mm_popcnt_u32((int) loadmask3); - // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) - // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) - // write out the job state for the lanes that are done (we need the final 'job3.out' value to compute the compressed string length) - _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; - _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; - _mm512_mask_compressstoreu_epi64(output, loadmask3, job3); output += delta3; diff --git a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll4.inc b/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll4.inc deleted file mode 100644 index 15cca7c938b4..000000000000 --- a/cpp/src/parquet/thirdparty/fsst/fsst_avx512_unroll4.inc +++ /dev/null @@ -1,228 +0,0 @@ -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// -// -// -// -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// -// -// -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// furnished to do so, subject to the following conditions: -// furnished to do so, subject to the following conditions: -// furnished to do so, subject to the following conditions: -// -// -// -// -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// -// -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E3PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E4PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// -// -// -// -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -// -// -// -// - // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). - // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). - // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask3=11111111, delta3=8). - // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask4=11111111, delta4=8). - job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; - job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; - job3 = _mm512_mask_expandloadu_epi64(job3, loadmask3, input); input += delta3; - job4 = _mm512_mask_expandloadu_epi64(job4, loadmask4, input); input += delta4; - // load the next 8 input string bytes (uncompressed data, aka 'symbols'). - // load the next 8 input string bytes (uncompressed data, aka 'symbols'). - // load the next 8 input string bytes (uncompressed data, aka 'symbols'). - // load the next 8 input string bytes (uncompressed data, aka 'symbols'). - __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); - __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); - __m512i word3 = _mm512_i64gather_epi64(_mm512_srli_epi64(job3, 46), symbolBase, 1); - __m512i word4 = _mm512_i64gather_epi64(_mm512_srli_epi64(job4, 46), symbolBase, 1); - // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. - // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. - // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. - // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. - // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). - // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). - // code3: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). - // code4: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). - __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); - __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); - __m512i code3 = _mm512_i64gather_epi64(_mm512_and_epi64(word3, all_FFFF), symbolTable.shortCodes, sizeof(u16)); - __m512i code4 = _mm512_i64gather_epi64(_mm512_and_epi64(word4, all_FFFF), symbolTable.shortCodes, sizeof(u16)); - // get the first three bytes of the string. - // get the first three bytes of the string. - // get the first three bytes of the string. - // get the first three bytes of the string. - __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); - __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); - __m512i pos3 = _mm512_mullo_epi64(_mm512_and_epi64(word3, all_FFFFFF), all_PRIME); - __m512i pos4 = _mm512_mullo_epi64(_mm512_and_epi64(word4, all_FFFFFF), all_PRIME); - // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT - // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT - // hash them into a random number: pos3 = pos3*PRIME; pos3 ^= pos3>>SHIFT - // hash them into a random number: pos4 = pos4*PRIME; pos4 ^= pos4>>SHIFT - pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); - pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); - pos3 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos3,_mm512_srli_epi64(pos3,FSST_SHIFT)), all_HASH), 4); - pos4 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos4,_mm512_srli_epi64(pos4,FSST_SHIFT)), all_HASH), 4); - // lookup in the 3-byte-prefix keyed hash table - // lookup in the 3-byte-prefix keyed hash table - // lookup in the 3-byte-prefix keyed hash table - // lookup in the 3-byte-prefix keyed hash table - __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); - __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); - __m512i icl3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 8), 1); - __m512i icl4 = _mm512_i64gather_epi64(pos4, (((char*) symbolTable.hashTab) + 8), 1); - // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). - // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). - // speculatively store the first input byte into the second position of the write3 register (in case it turns out to be an escaped byte). - // speculatively store the first input byte into the second position of the write4 register (in case it turns out to be an escaped byte). - __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); - __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); - __m512i write3 = _mm512_slli_epi64(_mm512_and_epi64(word3, all_FF), 8); - __m512i write4 = _mm512_slli_epi64(_mm512_and_epi64(word4, all_FF), 8); - // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. - // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. - // lookup just like the icl3 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. - // lookup just like the icl4 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. - __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); - __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); - __m512i symb3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 0), 1); - __m512i symb4 = _mm512_i64gather_epi64(pos4, (((char*) symbolTable.hashTab) + 0), 1); - // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). - // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). - // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). - // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). - pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); - pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); - pos3 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl3, all_FF)); - pos4 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl4, all_FF)); - // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). - // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). - // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). - // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). - __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); - __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); - __mmask8 match3 = _mm512_cmpeq_epi64_mask(symb3, _mm512_and_epi64(word3, pos3)) & _mm512_cmplt_epi64_mask(icl3, all_ICL_FREE); - __mmask8 match4 = _mm512_cmpeq_epi64_mask(symb4, _mm512_and_epi64(word4, pos4)) & _mm512_cmplt_epi64_mask(icl4, all_ICL_FREE); - // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. - // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. - // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. - // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. - code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); - code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); - code3 = _mm512_mask_mov_epi64(code3, match3, _mm512_srli_epi64(icl3, 16)); - code4 = _mm512_mask_mov_epi64(code4, match4, _mm512_srli_epi64(icl4, 16)); - // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. - // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. - // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. - // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. - write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); - write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); - write3 = _mm512_or_epi64(write3, _mm512_and_epi64(code3, all_FF)); - write4 = _mm512_or_epi64(write4, _mm512_and_epi64(code4, all_FF)); - // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) - // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) - // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) - // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) - code1 = _mm512_and_epi64(code1, all_FFFF); - code2 = _mm512_and_epi64(code2, all_FFFF); - code3 = _mm512_and_epi64(code3, all_FFFF); - code4 = _mm512_and_epi64(code4, all_FFFF); - // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) - // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) - // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) - // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) - _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); - _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); - _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job3, all_M19), write3, 1); - _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job4, all_M19), write4, 1); - // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) - // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) - // increase the job3.cur field in the job with the symbol length (for this, shift away 12 bits from the code) - // increase the job4.cur field in the job with the symbol length (for this, shift away 12 bits from the code) - job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); - job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); - job3 = _mm512_add_epi64(job3, _mm512_slli_epi64(_mm512_srli_epi64(code3, FSST_LEN_BITS), 46)); - job4 = _mm512_add_epi64(job4, _mm512_slli_epi64(_mm512_srli_epi64(code4, FSST_LEN_BITS), 46)); - // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) - // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) - // increase the job3.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) - // increase the job4.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) - job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); - job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); - job3 = _mm512_add_epi64(job3, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code3, 8), all_ONE))); - job4 = _mm512_add_epi64(job4, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code4, 8), all_ONE))); - // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) - // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) - // test which lanes are done now (job3.cur==job3.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job3 register) - // test which lanes are done now (job4.cur==job4.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job4 register) - loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); - loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); - loadmask3 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job3, 46), _mm512_and_epi64(_mm512_srli_epi64(job3, 28), all_M18)); - loadmask4 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job4, 46), _mm512_and_epi64(_mm512_srli_epi64(job4, 28), all_M18)); - // calculate the amount of lanes in job1 that are done - // calculate the amount of lanes in job2 that are done - // calculate the amount of lanes in job3 that are done - // calculate the amount of lanes in job4 that are done - delta1 = _mm_popcnt_u32((int) loadmask1); - delta2 = _mm_popcnt_u32((int) loadmask2); - delta3 = _mm_popcnt_u32((int) loadmask3); - delta4 = _mm_popcnt_u32((int) loadmask4); - // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) - // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) - // write out the job state for the lanes that are done (we need the final 'job3.out' value to compute the compressed string length) - // write out the job state for the lanes that are done (we need the final 'job4.out' value to compute the compressed string length) - _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; - _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; - _mm512_mask_compressstoreu_epi64(output, loadmask3, job3); output += delta3; - _mm512_mask_compressstoreu_epi64(output, loadmask4, job4); output += delta4; diff --git a/cpp/src/parquet/thirdparty/fsst/libfsst.cpp b/cpp/src/parquet/thirdparty/fsst/libfsst.cpp deleted file mode 100644 index e3ba787b9592..000000000000 --- a/cpp/src/parquet/thirdparty/fsst/libfsst.cpp +++ /dev/null @@ -1,651 +0,0 @@ -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -#include "libfsst.hpp" - -namespace libfsst { -Symbol concat(Symbol a, Symbol b) { - Symbol s; - u32 length = a.length()+b.length(); - if (length > Symbol::maxLength) length = Symbol::maxLength; - s.set_code_len(FSST_CODE_MASK, length); - s.store_num((b.load_num() << (8*a.length())) | a.load_num()); - return s; -} -} // namespace libfsst - -namespace std { -template <> -class hash { - public: - size_t operator()(const libfsst::QSymbol& q) const { - uint64_t k = q.symbol.load_num(); - const uint64_t m = 0xc6a4a7935bd1e995; - const int r = 47; - uint64_t h = 0x8445d61a4e774912 ^ (8*m); - k *= m; - k ^= k >> r; - k *= m; - h ^= k; - h *= m; - h ^= h >> r; - h *= m; - h ^= h >> r; - return h; - } -}; -} - -namespace libfsst { -bool isEscapeCode(u16 pos) { return pos < FSST_CODE_BASE; } - -std::ostream& operator<<(std::ostream& out, const Symbol& s) { - for (u32 i=0; i line, const size_t len[], bool zeroTerminated=false) { - SymbolTable *st = new SymbolTable(), *bestTable = new SymbolTable(); - int bestGain = (int) -FSST_SAMPLEMAXSZ; // worst case (everything exception) - size_t sampleFrac = 128; - - // start by determining the terminator. We use the (lowest) most infrequent byte as terminator - st->zeroTerminated = zeroTerminated; - if (zeroTerminated) { - st->terminator = 0; // except in case of zeroTerminated mode, then byte 0 is terminator regardless frequency - } else { - u16 byteHisto[256]; - memset(byteHisto, 0, sizeof(byteHisto)); - for(size_t i=0; iterminator = 256; - while(i-- > 0) { - if (byteHisto[i] > minSize) continue; - st->terminator = i; - minSize = byteHisto[i]; - } - } - assert(st->terminator != 256); - - // a random number between 0 and 128 - auto rnd128 = [&](size_t i) { return 1 + (FSST_HASH((i+1UL)*sampleFrac)&127); }; - - // compress sample, and compute (pair-)frequencies - auto compressCount = [&](SymbolTable *st, Counters &counters) { // returns gain - int gain = 0; - - for(size_t i=0; i sampleFrac) continue; - } - if (cur < end) { - u16 code2 = 255, code1 = st->findLongestSymbol(cur, end); - cur += st->symbols[code1].length(); - gain += (int) (st->symbols[code1].length()-(1+isEscapeCode(code1))); - while (true) { - // count single symbol (i.e. an option is not extending it) - counters.count1Inc(code1); - - // as an alternative, consider just using the next byte.. - if (st->symbols[code1].length() != 1) // .. but do not count single byte symbols doubly - counters.count1Inc(*start); - - if (cur==end) { - break; - } - - // now match a new symbol - start = cur; - if (curhashTabSize-1); - Symbol s = st->hashTab[idx]; - code2 = st->shortCodes[word & 0xFFFF] & FSST_CODE_MASK; - word &= (0xFFFFFFFFFFFFFFFF >> (u8) s.icl); - if ((s.icl < FSST_ICL_FREE) & (s.load_num() == word)) { - code2 = s.code(); - cur += s.length(); - } else if (code2 >= FSST_CODE_BASE) { - cur += 2; - } else { - code2 = st->byteCodes[word & 0xFF] & FSST_CODE_MASK; - cur += 1; - } - } else { - code2 = st->findLongestSymbol(cur, end); - cur += st->symbols[code2].length(); - } - - // compute compressed output size - gain += ((int) (cur-start))-(1+isEscapeCode(code2)); - - if (sampleFrac < 128) { // no need to count pairs in final round - // consider the symbol that is the concatenation of the two last symbols - counters.count2Inc(code1, code2); - - // as an alternative, consider just extending with the next byte.. - if ((cur-start) > 1) // ..but do not count single byte extensions doubly - counters.count2Inc(code1, *start); - } - code1 = code2; - } - } - } - return gain; - }; - - auto makeTable = [&](SymbolTable *st, Counters &counters) { - // hashmap of c (needed because we can generate duplicate candidates) - unordered_set cands; - - // artificially make terminater the most frequent symbol so it gets included - u16 terminator = st->nSymbols?FSST_CODE_BASE:st->terminator; - counters.count1Set(terminator,65535); - - auto addOrInc = [&](unordered_set &cands, Symbol s, u64 count) { - if (count < (5*sampleFrac)/128) return; // improves both compression speed (less candidates), but also quality!! - QSymbol q; - q.symbol = s; - q.gain = count * s.length(); - auto it = cands.find(q); - if (it != cands.end()) { - q.gain += (*it).gain; - cands.erase(*it); - } - cands.insert(q); - }; - - // add candidate symbols based on counted frequency - for (u32 pos1=0; pos1nSymbols; pos1++) { - u32 cnt1 = counters.count1GetNext(pos1); // may advance pos1!! - if (!cnt1) continue; - - // heuristic: promoting single-byte symbols (*8) helps reduce exception rates and increases [de]compression speed - Symbol s1 = st->symbols[pos1]; - addOrInc(cands, s1, ((s1.length()==1)?8LL:1LL)*cnt1); - - if (sampleFrac >= 128 || // last round we do not create new (combined) symbols - s1.length() == Symbol::maxLength || // symbol cannot be extended - s1.val.str[0] == st->terminator) { // multi-byte symbols cannot contain the terminator byte - continue; - } - for (u32 pos2=0; pos2nSymbols; pos2++) { - u32 cnt2 = counters.count2GetNext(pos1, pos2); // may advance pos2!! - if (!cnt2) continue; - - // create a new symbol - Symbol s2 = st->symbols[pos2]; - Symbol s3 = concat(s1, s2); - if (s2.val.str[0] != st->terminator) // multi-byte symbols cannot contain the terminator byte - addOrInc(cands, s3, cnt2); - } - } - - // insert candidates into priority queue (by gain) - auto cmpGn = [](const QSymbol& q1, const QSymbol& q2) { return (q1.gain < q2.gain) || (q1.gain == q2.gain && q1.symbol.load_num() > q2.symbol.load_num()); }; - priority_queue,decltype(cmpGn)> pq(cmpGn); - for (auto& q : cands) - pq.push(q); - - // Create new symbol map using best candidates - st->clear(); - while (st->nSymbols < 255 && !pq.empty()) { - QSymbol q = pq.top(); - pq.pop(); - st->add(q.symbol); - } - }; - - u8 bestCounters[512*sizeof(u16)]; -#ifdef NONOPT_FSST - for(size_t frac : {127, 127, 127, 127, 127, 127, 127, 127, 127, 128}) { - sampleFrac = frac; -#else - for(sampleFrac=8; true; sampleFrac += 30) { -#endif - memset(&counters, 0, sizeof(Counters)); - long gain = compressCount(st, counters); - if (gain >= bestGain) { // a new best solution! - counters.backup1(bestCounters); - *bestTable = *st; bestGain = gain; - } - if (sampleFrac >= 128) break; // we do 5 rounds (sampleFrac=8,38,68,98,128) - makeTable(st, counters); - } - delete st; - counters.restore1(bestCounters); - makeTable(bestTable, counters); - bestTable->finalize(zeroTerminated); // renumber codes for more efficient compression - return bestTable; -} - -#ifndef NONOPT_FSST -static inline size_t compressSIMD(SymbolTable &symbolTable, u8* symbolBase, size_t nlines, const size_t len[], const u8* line[], size_t size, u8* dst, size_t lenOut[], u8* strOut[], int unroll) { - size_t curLine = 0, inOff = 0, outOff = 0, batchPos = 0, empty = 0, budget = size; - u8 *lim = dst + size, *codeBase = symbolBase + (1<<18); // 512KB temp space for compressing 512 strings - SIMDjob input[512]; // combined offsets of input strings (cur,end), and string #id (pos) and output (dst) pointer - SIMDjob output[512]; // output are (pos:9,dst:19) end pointers (compute compressed length from this) - size_t jobLine[512]; // for which line in the input sequence was this job (needed because we may split a line into multiple jobs) - - while (curLine < nlines && outOff <= (1<<19)) { - size_t prevLine = curLine, chunk, curOff = 0; - - // bail out if the output buffer cannot hold the compressed next string fully - if (((len[curLine]-curOff)*2 + 7) > budget) break; // see below for the +7 - else budget -= (len[curLine]-curOff)*2; - - strOut[curLine] = (u8*) 0; - lenOut[curLine] = 0; - - do { - do { - chunk = len[curLine] - curOff; - if (chunk > 511) { - chunk = 511; // large strings need to be chopped up into segments of 511 bytes - } - // create a job in this batch - SIMDjob job; - job.cur = inOff; - job.end = job.cur + chunk; - job.pos = batchPos; - job.out = outOff; - - // worst case estimate for compressed size (+7 is for the scatter that writes extra 7 zeros) - outOff += 7 + 2*(size_t)(job.end - job.cur); // note, total size needed is 512*(511*2+7) bytes. - if (outOff > (1<<19)) break; // simdbuf may get full, stop before this chunk - - // register job in this batch - input[batchPos] = job; - jobLine[batchPos] = curLine; - - if (chunk == 0) { - empty++; // detect empty chunks -- SIMD code cannot handle empty strings, so they need to be filtered out - } else { - // copy string chunk into temp buffer - memcpy(symbolBase + inOff, line[curLine] + curOff, chunk); - inOff += chunk; - curOff += chunk; - symbolBase[inOff++] = (u8) symbolTable.terminator; // write an extra char at the end that will not be encoded - } - if (++batchPos == 512) break; - } while(curOff < len[curLine]); - - if ((batchPos == 512) || (outOff > (1<<19)) || (++curLine >= nlines) || (((len[curLine])*2 + 7) > budget)) { // cannot accumulate more? - if (batchPos-empty >= 32) { // if we have enough work, fire off fsst_compressAVX512 (32 is due to max 4x8 unrolling) - // radix-sort jobs on length (longest string first) - // -- this provides best load balancing and allows to skip empty jobs at the end - u16 sortpos[513]; - memset(sortpos, 0, sizeof(sortpos)); - - // calculate length histo - for(size_t i=0; i> (u8) s.icl); - if ((s.icl < FSST_ICL_FREE) && s.load_num() == word) { - *out++ = (u8) s.code(); cur += s.length(); - } else { - // could be a 2-byte or 1-byte code, or miss - // handle everything with predication - *out = (u8) code; - out += 1+((code&FSST_CODE_BASE)>>8); - cur += (code>>FSST_LEN_BITS); - } - } - job.out = out - codeBase; - } - // postprocess job info - job.cur = 0; - job.end = job.out - input[job.pos].out; // misuse .end field as compressed size - job.out = input[job.pos].out; // reset offset to start of encoded string - input[job.pos] = job; - } - - // copy out the result data - for(size_t i=0; i> (u8) s.icl); - if ((s.icl < FSST_ICL_FREE) && s.load_num() == word) { - *out++ = (u8) s.code(); cur += s.length(); - } else if (avoidBranch) { - // could be a 2-byte or 1-byte code, or miss - // handle everything with predication - *out = (u8) code; - out += 1+((code&FSST_CODE_BASE)>>8); - cur += (code>>FSST_LEN_BITS); - } else if ((u8) code < byteLim) { - // 2 byte code after checking there is no longer pattern - *out++ = (u8) code; cur += 2; - } else { - // 1 byte code or miss. - *out = (u8) code; - out += 1+((code&FSST_CODE_BASE)>>8); // predicated - tested with a branch, that was always worse - cur++; - } - } - } - }; - - for(curLine=0; curLine 511) { - chunk = 511; // we need to compress in chunks of 511 in order to be byte-compatible with simd-compressed FSST - } - if ((2*chunk+7) > (size_t) (lim-out)) { - return curLine; // out of memory - } - // copy the string to the 511-byte buffer - memcpy(buf, cur, chunk); - buf[chunk] = (u8) symbolTable.terminator; - cur = buf; - end = cur + chunk; - - // based on symboltable stats, choose a variant that is nice to the branch predictor - if (noSuffixOpt) { - compressVariant(true,false); - } else if (avoidBranch) { - compressVariant(false,true); - } else { - compressVariant(false, false); - } - } while((curOff += chunk) < lenIn[curLine]); - lenOut[curLine] = (size_t) (out - strOut[curLine]); - } - return curLine; -} - -#define FSST_SAMPLELINE ((size_t) 512) - -// quickly select a uniformly random set of lines such that we have between [FSST_SAMPLETARGET,FSST_SAMPLEMAXSZ) string bytes -vector makeSample(u8* sampleBuf, const u8* strIn[], const size_t **lenRef, size_t nlines) { - size_t totSize = 0; - const size_t *lenIn = *lenRef; - vector sample; - - for(size_t i=0; i sample = makeSample(sampleBuf, strIn, &sampleLen, n?n:1); // careful handling of input to get a right-size and representative sample - Encoder *encoder = new Encoder(); - encoder->symbolTable = shared_ptr(buildSymbolTable(encoder->counters, sample, sampleLen, zeroTerminated)); - if (sampleLen != lenIn) delete[] sampleLen; - delete[] sampleBuf; - return (fsst_encoder_t*) encoder; -} - -/* create another encoder instance, necessary to do multi-threaded encoding using the same symbol table */ -extern "C" fsst_encoder_t* fsst_duplicate(fsst_encoder_t *encoder) { - Encoder *e = new Encoder(); - e->symbolTable = ((Encoder*)encoder)->symbolTable; // it is a shared_ptr - return (fsst_encoder_t*) e; -} - -// export a symbol table in compact format. -extern "C" u32 fsst_export(fsst_encoder_t *encoder, u8 *buf) { - Encoder *e = (Encoder*) encoder; - // In ->version there is a versionnr, but we hide also suffixLim/terminator/nSymbols there. - // This is sufficient in principle to *reconstruct* a fsst_encoder_t from a fsst_decoder_t - // (such functionality could be useful to append compressed data to an existing block). - // - // However, the hash function in the encoder hash table is endian-sensitive, and given its - // 'lossy perfect' hashing scheme is *unable* to contain other-endian-produced symbol tables. - // Doing a endian-conversion during hashing will be slow and self-defeating. - // - // Overall, we could support reconstructing an encoder for incremental compression, but - // should enforce equal-endianness. Bit of a bummer. Not going there now. - // - // The version field is now there just for future-proofness, but not used yet - - // version allows keeping track of fsst versions, track endianness, and encoder reconstruction - u64 version = (FSST_VERSION << 32) | // version is 24 bits, most significant byte is 0 - (((u64) e->symbolTable->suffixLim) << 24) | - (((u64) e->symbolTable->terminator) << 16) | - (((u64) e->symbolTable->nSymbols) << 8) | - FSST_ENDIAN_MARKER; // least significant byte is nonzero - - version = swap64_if_be(version); // ensure version is little-endian encoded - - /* do not assume unaligned reads here */ - memcpy(buf, &version, 8); - buf[8] = e->symbolTable->zeroTerminated; - for(u32 i=0; i<8; i++) - buf[9+i] = (u8) e->symbolTable->lenHisto[i]; - u32 pos = 17; - - // emit only the used bytes of the symbols - for(u32 i = e->symbolTable->zeroTerminated; i < e->symbolTable->nSymbols; i++) - for(u32 j = 0; j < e->symbolTable->symbols[i].length(); j++) - buf[pos++] = e->symbolTable->symbols[i].val.str[j]; // serialize used symbol bytes - - return pos; // length of what was serialized -} - -#define FSST_CORRUPT 32774747032022883 /* 7-byte number in little endian containing "corrupt" */ - -extern "C" u32 fsst_import(fsst_decoder_t *decoder, u8 const *buf) { - u64 version = 0; - u32 code, pos = 17; - u8 lenHisto[8]; - - // version field (first 8 bytes) is now there just for future-proofness, unused still (skipped) - memcpy(&version, buf, 8); - version = swap64_if_be(version); // version is always little-endian encoded - - if ((version>>32) != FSST_VERSION) return 0; - decoder->zeroTerminated = buf[8]&1; - memcpy(lenHisto, buf+9, 8); - - // in case of zero-terminated, first symbol is "" (zero always, may be overwritten) - decoder->len[0] = 1; - decoder->symbol[0] = 0; - - // we use lenHisto[0] as 1-byte symbol run length (at the end) - code = decoder->zeroTerminated; - if (decoder->zeroTerminated) lenHisto[0]--; // if zeroTerminated, then symbol "" aka 1-byte code=0, is not stored at the end - - // now get all symbols from the buffer - for(u32 l=1; l<=8; l++) { /* l = 1,2,3,4,5,6,7,8 */ - for(u32 i=0; i < lenHisto[(l&7) /* 1,2,3,4,5,6,7,0 */]; i++, code++) { - decoder->len[code] = (l&7)+1; /* len = 2,3,4,5,6,7,8,1 */ - decoder->symbol[code] = 0; - for(u32 j=0; jlen[code]; j++) - ((u8*) &decoder->symbol[code])[j] = buf[pos++]; // note this enforces 'little endian' symbols - } - } - if (decoder->zeroTerminated) lenHisto[0]++; - - // fill unused symbols with text "corrupt". Gives a chance to detect corrupted code sequences (if there are unused symbols). - while(code<255) { - decoder->symbol[code] = FSST_CORRUPT; - decoder->len[code++] = 8; - } - return pos; -} - -// runtime check for simd -inline size_t _compressImpl(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd) { -#ifndef NONOPT_FSST - if (simd && fsst_hasAVX512()) - return compressSIMD(*e->symbolTable, e->simdbuf, nlines, lenIn, strIn, size, output, lenOut, strOut, simd); -#endif - (void) simd; - return compressBulk(*e->symbolTable, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch); -} -size_t compressImpl(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd) { - return _compressImpl(e, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch, simd); -} - -// adaptive choosing of scalar compression method based on symbol length histogram -inline size_t _compressAuto(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], int simd) { - bool avoidBranch = false, noSuffixOpt = false; - if (100*e->symbolTable->lenHisto[1] > 65*e->symbolTable->nSymbols && 100*e->symbolTable->suffixLim > 95*e->symbolTable->lenHisto[1]) { - noSuffixOpt = true; - } else if ((e->symbolTable->lenHisto[0] > 24 && e->symbolTable->lenHisto[0] < 92) && - (e->symbolTable->lenHisto[0] < 43 || e->symbolTable->lenHisto[6] + e->symbolTable->lenHisto[7] < 29) && - (e->symbolTable->lenHisto[0] < 72 || e->symbolTable->lenHisto[2] < 72)) { - avoidBranch = true; - } - return _compressImpl(e, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch, simd); -} -size_t compressAuto(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], int simd) { - return _compressAuto(e, nlines, lenIn, strIn, size, output, lenOut, strOut, simd); -} -} // namespace libfsst - -using namespace libfsst; -// the main compression function (everything automatic) -extern "C" size_t fsst_compress(fsst_encoder_t *encoder, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[]) { - // to be faster than scalar, simd needs 64 lines or more of length >=12; or fewer lines, but big ones (totLen > 32KB) - size_t totLen = accumulate(lenIn, lenIn+nlines, 0); - int simd = totLen > nlines*12 && (nlines > 64 || totLen > (size_t) 1<<15); - return _compressAuto((Encoder*) encoder, nlines, lenIn, strIn, size, output, lenOut, strOut, 3*simd); -} - -/* deallocate encoder */ -extern "C" void fsst_destroy(fsst_encoder_t* encoder) { - Encoder *e = (Encoder*) encoder; - delete e; -} - -/* very lazy implementation relying on export and import */ -extern "C" fsst_decoder_t fsst_decoder(fsst_encoder_t *encoder) { - u8 buf[sizeof(fsst_decoder_t)]; - u32 cnt1 = fsst_export(encoder, buf); - fsst_decoder_t decoder; - u32 cnt2 = fsst_import(&decoder, buf); - assert(cnt1 == cnt2); (void) cnt1; (void) cnt2; - return decoder; -} diff --git a/cpp/src/parquet/thirdparty/fsst/libfsst.hpp b/cpp/src/parquet/thirdparty/fsst/libfsst.hpp deleted file mode 100644 index e4d7c0aa9b9c..000000000000 --- a/cpp/src/parquet/thirdparty/fsst/libfsst.hpp +++ /dev/null @@ -1,474 +0,0 @@ -// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): -// -// Copyright 2018-2020, CWI, TU Munich, FSU Jena -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files -// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, -// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// -// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - -#include "fsst.h" // the official FSST API -- also usable by C mortals - -/* unsigned integers */ -namespace libfsst { -typedef uint8_t u8; -typedef uint16_t u16; -typedef uint32_t u32; -typedef uint64_t u64; -} // namespace libfsst - -#if UINTPTR_MAX == 0xffffffffU -// We're on a 32-bit platform -#define NONOPT_FSST -#endif - -#define FSST_ENDIAN_MARKER ((u64) 1) -#define FSST_VERSION_20190218 20190218 -#define FSST_VERSION ((u64) FSST_VERSION_20190218) - -// "symbols" are character sequences (up to 8 bytes) -// A symbol is compressed into a "code" of, in principle, one byte. But, we added an exception mechanism: -// byte 255 followed by byte X represents the single-byte symbol X. Its code is 256+X. - -// we represent codes in u16 (not u8). 12 bits code (of which 10 are used), 4 bits length -#define FSST_LEN_BITS 12 -#define FSST_CODE_BITS 9 -#define FSST_CODE_BASE 256UL /* first 256 codes [0,255] are pseudo codes: escaped bytes */ -#define FSST_CODE_MAX (1UL<=8) { - len = 8; - memcpy(val.str, input, 8); - } else { - memcpy(val.str, input, len); - } - set_code_len(FSST_CODE_MAX, len); - } - void set_code_len(u32 code, u32 len) { icl = (len<<28)|(code<<16)|((8-len)*8); } - - u64 load_num() const { return swap64_if_be(val.num); } - void store_num(u64 v) { val.num = swap64_if_be(v); } - - u32 length() const { return (u32) (icl >> 28); } - u16 code() const { return (icl >> 16) & FSST_CODE_MASK; } - u32 ignoredBits() const { return (u32) icl; } - - u8 first() const { assert( length() >= 1); return 0xFF & load_num(); } - u16 first2() const { assert( length() >= 2); return 0xFFFF & load_num(); } - -#define FSST_HASH_LOG2SIZE 10 -#define FSST_HASH_PRIME 2971215073LL -#define FSST_SHIFT 15 -#define FSST_HASH(w) (((w)*FSST_HASH_PRIME)^(((w)*FSST_HASH_PRIME)>>FSST_SHIFT)) - size_t hash() const { size_t v = 0xFFFFFF & load_num(); return FSST_HASH(v); } // hash on the next 3 bytes -}; - -// Symbol that can be put in a queue, ordered on gain -struct QSymbol{ - Symbol symbol; - mutable u32 gain; // mutable because gain value should be ignored in find() on unordered_set of QSymbols - bool operator==(const QSymbol& other) const { return symbol.val.num == other.symbol.val.num && symbol.length() == other.symbol.length(); } -}; - -// we construct FSST symbol tables using a random sample of about 16KB (1<<14) -#define FSST_SAMPLETARGET (1<<14) -#define FSST_SAMPLEMAXSZ ((long) 2*FSST_SAMPLETARGET) - -// two phases of compression, before and after optimize(): -// -// (1) to encode values we probe (and maintain) three datastructures: -// - u16 byteCodes[256] array at the position of the next byte (s.length==1) -// - u16 shortCodes[65536] array at the position of the next twobyte pattern (s.length==2) -// - Symbol hashtable[1024] (keyed by the next three bytes, ie for s.length>2), -// this search will yield a u16 code, it points into Symbol symbols[]. You always find a hit, because the first 256 codes are -// pseudo codes representing a single byte these will become escapes) -// -// (2) when we finished looking for the best symbol table we call optimize() to reshape it: -// - it renumbers the codes by length (first symbols of length 2,3,4,5,6,7,8; then 1 (starting from byteLim are symbols of length 1) -// length 2 codes for which no longer suffix symbol exists (< suffixLim) come first among the 2-byte codes -// (allows shortcut during compression) -// - for each two-byte combination, in all unused slots of shortCodes[], it enters the byteCode[] of the symbol corresponding -// to the first byte (if such a single-byte symbol exists). This allows us to just probe the next two bytes (if there is only one -// byte left in the string, there is still a terminator-byte added during compression) in shortCodes[]. That is, byteCodes[] -// and its codepath is no longer required. This makes compression faster. The reason we use byteCodes[] during symbolTable construction -// is that adding a new code/symbol is expensive (you have to touch shortCodes[] in 256 places). This optimization was -// hence added to make symbolTable construction faster. -// -// this final layout allows for the fastest compression code, only currently present in compressBulk - -// in the hash table, the icl field contains (low-to-high) ignoredBits:16,code:12,length:4 -#define FSST_ICL_FREE ((15<<28)|(((u32)FSST_CODE_MASK)<<16)) // high bits of icl (len=8,code=FSST_CODE_MASK) indicates free bucket - -// ignoredBits is (8-length)*8, which is the amount of high bits to zero in the input word before comparing with the hashtable key -// ..it could of course be computed from len during lookup, but storing it precomputed in some loose bits is faster -// -// the gain field is only used in the symbol queue that sorts symbols on gain - -struct SymbolTable { - static const u32 hashTabSize = 1<> (u8) s.icl)); - return true; - } - bool add(Symbol s) { - assert(FSST_CODE_BASE + nSymbols < FSST_CODE_MAX); - u32 len = s.length(); - s.set_code_len(FSST_CODE_BASE + nSymbols, len); - if (len == 1) { - byteCodes[s.first()] = FSST_CODE_BASE + nSymbols + (1<> ((u8) hashTab[idx].icl)))) { - return (hashTab[idx].icl>>16) & FSST_CODE_MASK; // matched a long symbol - } - if (s.length() >= 2) { - u16 code = shortCodes[s.first2()] & FSST_CODE_MASK; - if (code >= FSST_CODE_BASE) return code; - } - return byteCodes[s.first()] & FSST_CODE_MASK; - } - u16 findLongestSymbol(const u8* cur, const u8* end) const { - return findLongestSymbol(Symbol(cur,end)); // represent the string as a temporary symbol - } - - // rationale for finalize: - // - during symbol table construction, we may create more than 256 codes, but bring it down to max 255 in the last makeTable() - // consequently we needed more than 8 bits during symbol table contruction, but can simplify the codes to single bytes in finalize() - // (this feature is in fact lo longer used, but could still be exploited: symbol construction creates no more than 255 symbols in each pass) - // - we not only reduce the amount of codes to <255, but also *reorder* the symbols and renumber their codes, for higher compression perf. - // we renumber codes so they are grouped by length, to allow optimized scalar string compression (byteLim and suffixLim optimizations). - // - we make the use of byteCode[] no longer necessary by inserting single-byte codes in the free spots of shortCodes[] - // Using shortCodes[] only makes compression faster. When creating the symbolTable, however, using shortCodes[] for the single-byte - // symbols is slow, as each insert touches 256 positions in it. This optimization was added when optimizing symbolTable construction time. - // - // In all, we change the layout and coding, as follows.. - // - // before finalize(): - // - The real symbols are symbols[256..256+nSymbols>. As we may have nSymbols > 255 - // - The first 256 codes are pseudo symbols (all escaped bytes) - // - // after finalize(): - // - table layout is symbols[0..nSymbols>, with nSymbols < 256. - // - Real codes are [0,nSymbols>. 8-th bit not set. - // - Escapes in shortCodes have the 8th bit set (value: 256+255=511). 255 because the code to be emitted is the escape byte 255 - // - symbols are grouped by length: 2,3,4,5,6,7,8, then 1 (single-byte codes last) - // the two-byte codes are split in two sections: - // - first section contains codes for symbols for which there is no longer symbol (no suffix). It allows an early-out during compression - // - // finally, shortCodes[] is modified to also encode all single-byte symbols (hence byteCodes[] is not required on a critical path anymore). - // - void finalize(u8 zeroTerminated) { - assert(nSymbols <= 255); - u8 newCode[256], rsum[8], byteLim = nSymbols - (lenHisto[0] - zeroTerminated); - - // compute running sum of code lengths (starting offsets for each length) - rsum[0] = byteLim; // 1-byte codes are highest - rsum[1] = zeroTerminated; - for(u32 i=1; i<7; i++) - rsum[i+1] = rsum[i] + lenHisto[i]; - - // determine the new code for each symbol, ordered by length (and splitting 2byte symbols into two classes around suffixLim) - suffixLim = rsum[1]; - symbols[newCode[0] = 0] = symbols[256]; // keep symbol 0 in place (for zeroTerminated cases only) - - for(u32 i=zeroTerminated, j=rsum[2]; i 1 && first2 == s2.first2()) // test if symbol k is a suffix of s - opt = 0; - } - newCode[i] = opt?suffixLim++:--j; // symbols without a larger suffix have a code < suffixLim - } else - newCode[i] = rsum[len-1]++; - s1.set_code_len(newCode[i],len); - symbols[newCode[i]] = s1; - } - // renumber the codes in byteCodes[] - for(u32 i=0; i<256; i++) - if ((byteCodes[i] & FSST_CODE_MASK) >= FSST_CODE_BASE) - byteCodes[i] = newCode[(u8) byteCodes[i]] + (1 << FSST_LEN_BITS); - else - byteCodes[i] = 511 + (1 << FSST_LEN_BITS); - - // renumber the codes in shortCodes[] - for(u32 i=0; i<65536; i++) - if ((shortCodes[i] & FSST_CODE_MASK) >= FSST_CODE_BASE) - shortCodes[i] = newCode[(u8) shortCodes[i]] + (shortCodes[i] & (15 << FSST_LEN_BITS)); - else - shortCodes[i] = byteCodes[i&0xFF]; - - // replace the symbols in the hash table - for(u32 i=0; i>8; - } - void count1Inc(u32 pos1) { - if (!count1Low[pos1]++) // increment high early (when low==0, not when low==255). This means (high > 0) <=> (cnt > 0) - count1High[pos1]++; //(0,0)->(1,1)->..->(255,1)->(0,1)->(1,2)->(2,2)->(3,2)..(255,2)->(0,2)->(1,3)->(2,3)... - } - void count2Inc(u32 pos1, u32 pos2) { - if (!count2Low[pos1][pos2]++) // increment high early (when low==0, not when low==255). This means (high > 0) <=> (cnt > 0) - // inc 4-bits high counter with 1<<0 (1) or 1<<4 (16) -- depending on whether pos2 is even or odd, repectively - count2High[pos1][(pos2)>>1] += 1 << (((pos2)&1)<<2); // we take our chances with overflow.. (4K maxval, on a 8K sample) - } - u32 count1GetNext(u32 &pos1) { // note: we will advance pos1 to the next nonzero counter in register range - // read 16-bits single symbol counter, split into two 8-bits numbers (count1Low, count1High), while skipping over zeros - u64 high = fsst_unaligned_load(&count1High[pos1]); // note: this reads 8 subsequent counters [pos1..pos1+7] - - u32 zero = high?(__builtin_ctzl(high)>>3):7UL; // number of zero bytes - high = (high >> (zero << 3)) & 255; // advance to nonzero counter - if (((pos1 += zero) >= FSST_CODE_MAX) || !high) // SKIP! advance pos2 - return 0; // all zero - - u32 low = count1Low[pos1]; - if (low) high--; // high is incremented early and low late, so decrement high (unless low==0) - return (u32) ((high << 8) + low); - } - u32 count2GetNext(u32 pos1, u32 &pos2) { // note: we will advance pos2 to the next nonzero counter in register range - // read 12-bits pairwise symbol counter, split into low 8-bits and high 4-bits number while skipping over zeros - u64 high = fsst_unaligned_load(&count2High[pos1][pos2>>1]); // note: this reads 16 subsequent counters [pos2..pos2+15] - high >>= ((pos2&1) << 2); // odd pos2: ignore the lowest 4 bits & we see only 15 counters - - u32 zero = high?(__builtin_ctzl(high)>>2):(15UL-(pos2&1UL)); // number of zero 4-bits counters - high = (high >> (zero << 2)) & 15; // advance to nonzero counter - if (((pos2 += zero) >= FSST_CODE_MAX) || !high) // SKIP! advance pos2 - return 0UL; // all zero - - u32 low = count2Low[pos1][pos2]; - if (low) high--; // high is incremented early and low late, so decrement high (unless low==0) - return (u32) ((high << 8) + low); - } - void backup1(u8 *buf) { - memcpy(buf, count1High, FSST_CODE_MAX); - memcpy(buf+FSST_CODE_MAX, count1Low, FSST_CODE_MAX); - } - void restore1(u8 *buf) { - memcpy(count1High, buf, FSST_CODE_MAX); - memcpy(count1Low, buf+FSST_CODE_MAX, FSST_CODE_MAX); - } -}; -#endif - - -#define FSST_BUFSZ (3<<19) // 768KB - -// an encoder is a symbolmap plus some bufferspace, needed during map construction as well as compression -struct Encoder { - shared_ptr symbolTable; // symbols, plus metadata and data structures for quick compression (shortCode,hashTab, etc) - union { - Counters counters; // for counting symbol occurences during map construction - u8 simdbuf[FSST_BUFSZ]; // for compression: SIMD string staging area 768KB = 256KB in + 512KB out (worst case for 256KB in) - }; -}; - -// job control integer representable in one 64bits SIMD lane: cur/end=input, out=output, pos=which string (2^9=512 per call) -struct SIMDjob { - u64 out:19,pos:9,end:18,cur:18; // cur/end is input offsets (2^18=256KB), out is output offset (2^19=512KB) -}; - -extern bool -fsst_hasAVX512(); // runtime check for avx512 capability - -extern size_t -fsst_compressAVX512( - SymbolTable &symbolTable, - u8* codeBase, // IN: base address for codes, i.e. compression output (points to simdbuf+256KB) - u8* symbolBase, // IN: base address for string bytes, i.e. compression input (points to simdbuf) - SIMDjob* input, // IN: input array (size n) with job information: what to encode, where to store it. - SIMDjob* output, // OUT: output array (size n) with job information: how much got encoded, end output pointer. - size_t n, // IN: size of arrays input and output (should be max 512) - size_t unroll); // IN: degree of SIMD unrolling - -// Symbol manipulation -Symbol concat(Symbol a, Symbol b); - -// C++ fsst-compress function with some more control of how the compression happens (algorithm flavor, simd unroll degree) -size_t compressImpl(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd); -size_t compressAuto(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], int simd); -} // namespace libfsst diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 7ba1f4f876be..ad84ffc98ba2 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -66,6 +66,8 @@ ARROW_CARES_BUILD_VERSION=1.17.2 ARROW_CARES_BUILD_SHA256_CHECKSUM=4803c844ce20ce510ef0eb83f8ea41fa24ecaae9d280c468c582d2bb25b3913d ARROW_CRC32C_BUILD_VERSION=1.1.2 ARROW_CRC32C_BUILD_SHA256_CHECKSUM=ac07840513072b7fcebda6e821068aa04889018f24e10e46181068fb214d7e56 +ARROW_FSST_BUILD_VERSION=89f49c580c6388acf3b6ed2a49e1bfde6c05e616 +ARROW_FSST_BUILD_SHA256_CHECKSUM=5921d2b837800e1c886903d18971994587328b3e5d08d32eb8e80c00890c304d ARROW_GBENCHMARK_BUILD_VERSION=v1.8.3 ARROW_GBENCHMARK_BUILD_SHA256_CHECKSUM=6bc180a57d23d4d9515519f92b0c83d61b05b5bab188961f36ac7b06b0d9e9ce ARROW_GFLAGS_BUILD_VERSION=v2.2.2 @@ -144,6 +146,7 @@ DEPENDENCIES=( "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz" "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://github.com/c-ares/c-ares/releases/download/cares-${ARROW_CARES_BUILD_VERSION//./_}/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz" "ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/refs/tags/${ARROW_CRC32C_BUILD_VERSION}.tar.gz" + "ARROW_FSST_URL fsst-${ARROW_FSST_BUILD_VERSION}.tar.gz https://github.com/cwida/fsst/archive/${ARROW_FSST_BUILD_VERSION}.tar.gz" "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz" "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz" "ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz" From 68cc6f236b0f3a8f1f4996800d556207827fe2aa Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 24 Nov 2025 08:36:45 +0000 Subject: [PATCH 04/24] update --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 8d67849ea478..2ce366f6c9ae 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2624,15 +2624,16 @@ function(build_fsst) URL_HASH "SHA256=${ARROW_FSST_BUILD_SHA256_CHECKSUM}") prepare_fetchcontent() - fetchcontent_makeavailable(fsst) + fetchcontent_getproperties(fsst) + if(NOT fsst_POPULATED) + fetchcontent_populate(fsst) + endif() - set(ARROW_FSST_INCLUDE_DIR - "${fsst_SOURCE_DIR}" - CACHE INTERNAL "FSST include directory") + set(ARROW_FSST_INCLUDE_DIR "${fsst_SOURCE_DIR}" PARENT_SCOPE) set(ARROW_FSST_SOURCES "${fsst_SOURCE_DIR}/libfsst.cpp;${fsst_SOURCE_DIR}/fsst_avx512.cpp" - CACHE INTERNAL "FSST source files") - set(FSST_VENDORED TRUE CACHE INTERNAL "Whether FSST is built from source") + PARENT_SCOPE) + set(FSST_VENDORED TRUE PARENT_SCOPE) endfunction() if(ARROW_WITH_FSST) From 05506fab80de3293dc1363366694e9c2792c4ecd Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 24 Nov 2025 08:54:13 +0000 Subject: [PATCH 05/24] update --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 2ce366f6c9ae..ada1654484ac 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2637,6 +2637,9 @@ function(build_fsst) endfunction() if(ARROW_WITH_FSST) + if("${fsst_SOURCE}" STREQUAL "") + set(fsst_SOURCE "BUNDLED") + endif() resolve_dependency(fsst IS_RUNTIME_DEPENDENCY FALSE) endif() From 5935b9e47abba492a5f013c7aa3b1cfb53ffbf61 Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 24 Nov 2025 09:31:16 +0000 Subject: [PATCH 06/24] update --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index ada1654484ac..ceb2c20ac498 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2637,9 +2637,7 @@ function(build_fsst) endfunction() if(ARROW_WITH_FSST) - if("${fsst_SOURCE}" STREQUAL "") - set(fsst_SOURCE "BUNDLED") - endif() + set(fsst_SOURCE "BUNDLED" CACHE STRING "Source of fsst dependency" FORCE) resolve_dependency(fsst IS_RUNTIME_DEPENDENCY FALSE) endif() From ff3b8f73ebfab93a30333d6a8bd8c00fb85d5593 Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 24 Nov 2025 10:32:46 +0000 Subject: [PATCH 07/24] update --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index ceb2c20ac498..245868948b89 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -73,6 +73,8 @@ set(ARROW_THIRDPARTY_DEPENDENCIES ZLIB zstd) +set(fsst_SOURCE "BUNDLED" CACHE STRING "Source of fsst dependency") + # For backward compatibility. We use "BOOST_SOURCE" if "Boost_SOURCE" # isn't specified and "BOOST_SOURCE" is specified. # We renamed "BOOST" dependency name to "Boost" in 3.0.0 because @@ -2637,7 +2639,9 @@ function(build_fsst) endfunction() if(ARROW_WITH_FSST) - set(fsst_SOURCE "BUNDLED" CACHE STRING "Source of fsst dependency" FORCE) + if(NOT fsst_SOURCE STREQUAL "BUNDLED") + message(FATAL_ERROR "FSST must currently be built from source. Set fsst_SOURCE=BUNDLED.") + endif() resolve_dependency(fsst IS_RUNTIME_DEPENDENCY FALSE) endif() From f5e2bdba8fbd6e5a21bc786d2474108a0873cf8c Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 24 Nov 2025 11:12:49 +0000 Subject: [PATCH 08/24] update --- cpp/src/parquet/CMakeLists.txt | 4 ++++ cpp/src/parquet/encoding_test.cc | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index a2d300d684da..66c9d08cb769 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -205,6 +205,10 @@ if(DEFINED ARROW_FSST_INCLUDE_DIR) list(APPEND PARQUET_PRIVATE_INCLUDE_DIRS ${ARROW_FSST_INCLUDE_DIR}) list(APPEND PARQUET_TEST_EXTRA_INCLUDES ${ARROW_FSST_INCLUDE_DIR}) endif() +if(DEFINED ARROW_FSST_SOURCES) + set_source_files_properties(${ARROW_FSST_SOURCES} + PROPERTIES COMPILE_OPTIONS "-Wno-error=shorten-64-to-32") +endif() if(ARROW_HAVE_RUNTIME_AVX2) # AVX2 is used as a proxy for BMI2. diff --git a/cpp/src/parquet/encoding_test.cc b/cpp/src/parquet/encoding_test.cc index a43bafd0960b..ebc86f148537 100644 --- a/cpp/src/parquet/encoding_test.cc +++ b/cpp/src/parquet/encoding_test.cc @@ -2886,7 +2886,7 @@ TEST(TestFsstEncoding, HeavyNullsDecodeSpaced) { } writer.Finish(); - const int null_count = binary.null_count(); + const int null_count = static_cast(binary.null_count()); ASSERT_EQ(static_cast(values->length() - null_count), decoder->DecodeSpaced(decoded.data(), static_cast(values->length()), null_count, valid_bits.data(), 0)); From 7c105d4d04b56101a1294971f170d90cc5dd38f1 Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 24 Nov 2025 11:47:43 +0000 Subject: [PATCH 09/24] update --- cpp/src/parquet/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 66c9d08cb769..38eb0dc59f1e 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -207,7 +207,9 @@ if(DEFINED ARROW_FSST_INCLUDE_DIR) endif() if(DEFINED ARROW_FSST_SOURCES) set_source_files_properties(${ARROW_FSST_SOURCES} - PROPERTIES COMPILE_OPTIONS "-Wno-error=shorten-64-to-32") + PROPERTIES + COMPILE_OPTIONS + "$<$:-Wno-error=shorten-64-to-32>") endif() if(ARROW_HAVE_RUNTIME_AVX2) From 511776f0b43697af94d12de760061a107c99368a Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 24 Nov 2025 14:22:07 +0000 Subject: [PATCH 10/24] update --- cpp/src/parquet/CMakeLists.txt | 9 +++++---- cpp/src/parquet/fsst_compat.h | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 cpp/src/parquet/fsst_compat.h diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 38eb0dc59f1e..e2b1db096339 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -206,10 +206,11 @@ if(DEFINED ARROW_FSST_INCLUDE_DIR) list(APPEND PARQUET_TEST_EXTRA_INCLUDES ${ARROW_FSST_INCLUDE_DIR}) endif() if(DEFINED ARROW_FSST_SOURCES) - set_source_files_properties(${ARROW_FSST_SOURCES} - PROPERTIES - COMPILE_OPTIONS - "$<$:-Wno-error=shorten-64-to-32>") + set_property(SOURCE ${ARROW_FSST_SOURCES} APPEND PROPERTY COMPILE_OPTIONS + "$<$,$>:-Wno-error=shorten-64-to-32;-Wno-shorten-64-to-32>" + "$<$,$,$>:-Wno-error=missing-declarations;-Wno-missing-declarations>" + "$<$:/wd4244>" + "$<$,$>>:-include${CMAKE_CURRENT_SOURCE_DIR}/fsst_compat.h>") endif() if(ARROW_HAVE_RUNTIME_AVX2) diff --git a/cpp/src/parquet/fsst_compat.h b/cpp/src/parquet/fsst_compat.h new file mode 100644 index 000000000000..2da27d273011 --- /dev/null +++ b/cpp/src/parquet/fsst_compat.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +// Provide minimal compatibility shims so third-party FSST sources +// can be compiled with the compilers Arrow supports. + +#if defined(_WIN32) && !defined(_MSC_VER) +#include + +// MinGW does not provide __cpuidex, but FSST only needs the CPUID +// leaf/sub-leaf variant that __cpuid_count implements. +static inline void arrow_fsst_cpuidex(int info[4], int function_id, int subfunction_id) { + __cpuid_count(function_id, subfunction_id, info[0], info[1], info[2], info[3]); +} +#define __cpuidex(info, function_id, subfunction_id) \ + arrow_fsst_cpuidex(info, function_id, subfunction_id) +#endif From 0344d45478bf593af40c9c2543065ed6014dbd41 Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 24 Nov 2025 14:41:34 +0000 Subject: [PATCH 11/24] update --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 245868948b89..39bda3d26686 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -647,8 +647,7 @@ if(DEFINED ENV{ARROW_FSST_URL}) set(FSST_SOURCE_URL "$ENV{ARROW_FSST_URL}") else() set_urls(FSST_SOURCE_URL - "https://github.com/cwida/fsst/archive/${ARROW_FSST_BUILD_VERSION}.tar.gz" - "${THIRDPARTY_MIRROR_URL}/fsst-${ARROW_FSST_BUILD_VERSION}.tar.gz") + "https://github.com/cwida/fsst/archive/${ARROW_FSST_BUILD_VERSION}.tar.gz") endif() if(DEFINED ENV{ARROW_GBENCHMARK_URL}) From f9b43c48f2a1b596cb080378dc7fe146a7f30e80 Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 24 Nov 2025 15:52:53 +0000 Subject: [PATCH 12/24] update --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 26 +++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 39bda3d26686..33db56e1969b 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -643,11 +643,14 @@ else() ) endif() +set(FSST_SOURCE_URL "") +set(FSST_GIT_REPOSITORY "") if(DEFINED ENV{ARROW_FSST_URL}) set(FSST_SOURCE_URL "$ENV{ARROW_FSST_URL}") +elseif(DEFINED ENV{ARROW_FSST_GIT_REPOSITORY}) + set(FSST_GIT_REPOSITORY "$ENV{ARROW_FSST_GIT_REPOSITORY}") else() - set_urls(FSST_SOURCE_URL - "https://github.com/cwida/fsst/archive/${ARROW_FSST_BUILD_VERSION}.tar.gz") + set(FSST_GIT_REPOSITORY "https://github.com/cwida/fsst.git") endif() if(DEFINED ENV{ARROW_GBENCHMARK_URL}) @@ -2620,9 +2623,22 @@ endif() function(build_fsst) message(STATUS "Building FSST from source using FetchContent") - fetchcontent_declare(fsst - URL ${FSST_SOURCE_URL} - URL_HASH "SHA256=${ARROW_FSST_BUILD_SHA256_CHECKSUM}") + if(FSST_SOURCE_URL) + fetchcontent_declare(fsst + ${FC_DECLARE_COMMON_OPTIONS} + URL ${FSST_SOURCE_URL} + URL_HASH "SHA256=${ARROW_FSST_BUILD_SHA256_CHECKSUM}") + else() + if(NOT FSST_GIT_REPOSITORY) + message(FATAL_ERROR "FSST_GIT_REPOSITORY is not set and no FSST_SOURCE_URL override was provided.") + endif() + fetchcontent_declare(fsst + ${FC_DECLARE_COMMON_OPTIONS} + GIT_REPOSITORY ${FSST_GIT_REPOSITORY} + GIT_TAG ${ARROW_FSST_BUILD_VERSION} + GIT_SHALLOW TRUE + GIT_PROGRESS TRUE) + endif() prepare_fetchcontent() fetchcontent_getproperties(fsst) From c49f19c771eaee4d46a56eaecde0df625adfd58f Mon Sep 17 00:00:00 2001 From: arnavb Date: Tue, 25 Nov 2025 07:31:32 +0000 Subject: [PATCH 13/24] vendor fsst --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 39 +- cpp/thirdparty/fsst/fsst.h | 227 +++++++ cpp/thirdparty/fsst/fsst_avx512.cpp | 150 +++++ cpp/thirdparty/fsst/fsst_avx512_unroll1.inc | 57 ++ cpp/thirdparty/fsst/fsst_avx512_unroll2.inc | 114 ++++ cpp/thirdparty/fsst/fsst_avx512_unroll3.inc | 171 +++++ cpp/thirdparty/fsst/fsst_avx512_unroll4.inc | 228 +++++++ cpp/thirdparty/fsst/libfsst.cpp | 651 ++++++++++++++++++++ cpp/thirdparty/fsst/libfsst.hpp | 471 ++++++++++++++ 9 files changed, 2072 insertions(+), 36 deletions(-) create mode 100644 cpp/thirdparty/fsst/fsst.h create mode 100644 cpp/thirdparty/fsst/fsst_avx512.cpp create mode 100644 cpp/thirdparty/fsst/fsst_avx512_unroll1.inc create mode 100644 cpp/thirdparty/fsst/fsst_avx512_unroll2.inc create mode 100644 cpp/thirdparty/fsst/fsst_avx512_unroll3.inc create mode 100644 cpp/thirdparty/fsst/fsst_avx512_unroll4.inc create mode 100644 cpp/thirdparty/fsst/libfsst.cpp create mode 100644 cpp/thirdparty/fsst/libfsst.hpp diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 33db56e1969b..c0c84511c1d5 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -643,16 +643,6 @@ else() ) endif() -set(FSST_SOURCE_URL "") -set(FSST_GIT_REPOSITORY "") -if(DEFINED ENV{ARROW_FSST_URL}) - set(FSST_SOURCE_URL "$ENV{ARROW_FSST_URL}") -elseif(DEFINED ENV{ARROW_FSST_GIT_REPOSITORY}) - set(FSST_GIT_REPOSITORY "$ENV{ARROW_FSST_GIT_REPOSITORY}") -else() - set(FSST_GIT_REPOSITORY "https://github.com/cwida/fsst.git") -endif() - if(DEFINED ENV{ARROW_GBENCHMARK_URL}) set(GBENCHMARK_SOURCE_URL "$ENV{ARROW_GBENCHMARK_URL}") else() @@ -2621,34 +2611,11 @@ if(ARROW_USE_XSIMD) endif() function(build_fsst) - message(STATUS "Building FSST from source using FetchContent") - - if(FSST_SOURCE_URL) - fetchcontent_declare(fsst - ${FC_DECLARE_COMMON_OPTIONS} - URL ${FSST_SOURCE_URL} - URL_HASH "SHA256=${ARROW_FSST_BUILD_SHA256_CHECKSUM}") - else() - if(NOT FSST_GIT_REPOSITORY) - message(FATAL_ERROR "FSST_GIT_REPOSITORY is not set and no FSST_SOURCE_URL override was provided.") - endif() - fetchcontent_declare(fsst - ${FC_DECLARE_COMMON_OPTIONS} - GIT_REPOSITORY ${FSST_GIT_REPOSITORY} - GIT_TAG ${ARROW_FSST_BUILD_VERSION} - GIT_SHALLOW TRUE - GIT_PROGRESS TRUE) - endif() - - prepare_fetchcontent() - fetchcontent_getproperties(fsst) - if(NOT fsst_POPULATED) - fetchcontent_populate(fsst) - endif() + message(STATUS "Configuring vendored FSST sources") - set(ARROW_FSST_INCLUDE_DIR "${fsst_SOURCE_DIR}" PARENT_SCOPE) + set(ARROW_FSST_INCLUDE_DIR "${ARROW_SOURCE_DIR}/thirdparty/fsst" PARENT_SCOPE) set(ARROW_FSST_SOURCES - "${fsst_SOURCE_DIR}/libfsst.cpp;${fsst_SOURCE_DIR}/fsst_avx512.cpp" + "${ARROW_SOURCE_DIR}/thirdparty/fsst/libfsst.cpp;${ARROW_SOURCE_DIR}/thirdparty/fsst/fsst_avx512.cpp" PARENT_SCOPE) set(FSST_VENDORED TRUE PARENT_SCOPE) endfunction() diff --git a/cpp/thirdparty/fsst/fsst.h b/cpp/thirdparty/fsst/fsst.h new file mode 100644 index 000000000000..71085d57201d --- /dev/null +++ b/cpp/thirdparty/fsst/fsst.h @@ -0,0 +1,227 @@ +/* + * the API for FSST compression -- (c) Peter Boncz, Viktor Leis and Thomas Neumann (CWI, TU Munich), 2018-2019 + * + * =================================================================================================================================== + * this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): + * + * Copyright 2018-2020, CWI, TU Munich, FSU Jena + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + * IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * You can contact the authors via the FSST source repository : https://github.com/cwida/fsst + * =================================================================================================================================== + * + * FSST: Fast Static Symbol Table compression + * see the paper https://github.com/cwida/fsst/raw/master/fsstcompression.pdf + * + * FSST is a compression scheme focused on string/text data: it can compress strings from distributions with many different values (i.e. + * where dictionary compression will not work well). It allows *random-access* to compressed data: it is not block-based, so individual + * strings can be decompressed without touching the surrounding data in a compressed block. When compared to e.g. lz4 (which is + * block-based), FSST achieves similar decompression speed, (2x) better compression speed and 30% better compression ratio on text. + * + * FSST encodes strings also using a symbol table -- but it works on pieces of the string, as it maps "symbols" (1-8 byte sequences) + * onto "codes" (single-bytes). FSST can also represent a byte as an exception (255 followed by the original byte). Hence, compression + * transforms a sequence of bytes into a (supposedly shorter) sequence of codes or escaped bytes. These shorter byte-sequences could + * be seen as strings again and fit in whatever your program is that manipulates strings. + * + * useful property: FSST ensures that strings that are equal, are also equal in their compressed form. + * + * In this API, strings are considered byte-arrays (byte = unsigned char) and a batch of strings is represented as an array of + * unsigned char* pointers to their starts. A seperate length array (of unsigned int) denotes how many bytes each string consists of. + * + * This representation as unsigned char* pointers tries to assume as little as possible on the memory management of the program + * that calls this API, and is also intended to allow passing strings into this API without copying (even if you use C++ strings). + * + * We optionally support C-style zero-terminated strings (zero appearing only at the end). In this case, the compressed strings are + * also zero-terminated strings. In zero-terminated mode, the zero-byte at the end *is* counted in the string byte-length. + */ +#ifndef FSST_INCLUDED_H +#define FSST_INCLUDED_H + +#ifdef _MSC_VER +#define __restrict__ +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ +#define __ORDER_LITTLE_ENDIAN__ 2 +#include +static inline int __builtin_ctzl(unsigned long long x) { + unsigned long ret; + _BitScanForward64(&ret, x); + return (int)ret; +} +#endif + +#ifdef __cplusplus +#define FSST_FALLTHROUGH [[fallthrough]] +#include +extern "C" { +#else +#define FSST_FALLTHROUGH +#endif + +#include + +/* A compressed string is simply a string of 1-byte codes; except for code 255, which is followed by an uncompressed byte. */ +#define FSST_ESC 255 + +/* Data structure needed for compressing strings - use fsst_duplicate() to create thread-local copies. Use fsst_destroy() to free. */ +typedef void* fsst_encoder_t; /* opaque type - it wraps around a rather large (~900KB) C++ object */ + +/* Data structure needed for decompressing strings - read-only and thus can be shared between multiple decompressing threads. */ +typedef struct { + unsigned long long version; /* version id */ + unsigned char zeroTerminated; /* terminator is a single-byte code that does not appear in longer symbols */ + unsigned char len[255]; /* len[x] is the byte-length of the symbol x (1 < len[x] <= 8). */ + unsigned long long symbol[255]; /* symbol[x] contains in LITTLE_ENDIAN the bytesequence that code x represents (0 <= x < 255). */ +} fsst_decoder_t; + +/* Calibrate a FSST symboltable from a batch of strings (it is best to provide at least 16KB of data). */ +fsst_encoder_t* +fsst_create( + size_t n, /* IN: number of strings in batch to sample from. */ + const size_t lenIn[], /* IN: byte-lengths of the inputs */ + const unsigned char *strIn[], /* IN: string start pointers. */ + int zeroTerminated /* IN: whether input strings are zero-terminated. If so, encoded strings are as well (i.e. symbol[0]=""). */ +); + +/* Create another encoder instance, necessary to do multi-threaded encoding using the same symbol table. */ +fsst_encoder_t* +fsst_duplicate( + fsst_encoder_t *encoder /* IN: the symbol table to duplicate. */ +); + +#define FSST_MAXHEADER (8+1+8+2048+1) /* maxlen of deserialized fsst header, produced/consumed by fsst_export() resp. fsst_import() */ + +/* Space-efficient symbol table serialization (smaller than sizeof(fsst_decoder_t) - by saving on the unused bytes in symbols of len < 8). */ +unsigned int /* OUT: number of bytes written in buf, at most sizeof(fsst_decoder_t) */ +fsst_export( + fsst_encoder_t *encoder, /* IN: the symbol table to dump. */ + unsigned char *buf /* OUT: pointer to a byte-buffer where to serialize this symbol table. */ +); + +/* Deallocate encoder. */ +void +fsst_destroy(fsst_encoder_t*); + +/* Return a decoder structure from serialized format (typically used in a block-, file- or row-group header). */ +unsigned int /* OUT: number of bytes consumed in buf (0 on failure). */ +fsst_import( + fsst_decoder_t *decoder, /* IN: this symbol table will be overwritten. */ + unsigned char const *buf /* IN: pointer to a byte-buffer where fsst_export() serialized this symbol table. */ +); + +/* Return a decoder structure from an encoder. */ +fsst_decoder_t +fsst_decoder( + fsst_encoder_t *encoder +); + +/* Compress a batch of strings (on AVX512 machines best performance is obtained by compressing more than 32KB of string volume). */ +/* The output buffer must be large; at least "conservative space" (7+2*inputlength) for the first string for something to happen. */ +size_t /* OUT: the number of compressed strings (<=n) that fit the output buffer. */ +fsst_compress( + fsst_encoder_t *encoder, /* IN: encoder obtained from fsst_create(). */ + size_t nstrings, /* IN: number of strings in batch to compress. */ + const size_t lenIn[], /* IN: byte-lengths of the inputs */ + const unsigned char *strIn[], /* IN: input string start pointers. */ + size_t outsize, /* IN: byte-length of output buffer. */ + unsigned char *output, /* OUT: memory buffer to put the compressed strings in (one after the other). */ + size_t lenOut[], /* OUT: byte-lengths of the compressed strings. */ + unsigned char *strOut[] /* OUT: output string start pointers. Will all point into [output,output+size). */ +); + +/* Decompress a single string, inlined for speed. */ +inline size_t /* OUT: bytesize of the decompressed string. If > size, the decoded output is truncated to size. */ +fsst_decompress( + const fsst_decoder_t *decoder, /* IN: use this symbol table for compression. */ + size_t lenIn, /* IN: byte-length of compressed string. */ + const unsigned char *strIn, /* IN: compressed string. */ + size_t size, /* IN: byte-length of output buffer. */ + unsigned char *output /* OUT: memory buffer to put the decompressed string in. */ +) { + unsigned char*__restrict__ len = (unsigned char* __restrict__) decoder->len; + unsigned char*__restrict__ strOut = (unsigned char* __restrict__) output; + unsigned long long*__restrict__ symbol = (unsigned long long* __restrict__) decoder->symbol; + size_t code, posOut = 0, posIn = 0; +#ifndef FSST_MUST_ALIGN /* defining on platforms that require aligned memory access may help their performance */ +#define FSST_UNALIGNED_STORE(dst,src) memcpy((unsigned long long*) (dst), &(src), sizeof(unsigned long long)) +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + while (posOut+32 <= size && posIn+4 <= lenIn) { + unsigned int nextBlock, escapeMask; + memcpy(&nextBlock, strIn+posIn, sizeof(unsigned int)); + escapeMask = (nextBlock&0x80808080u)&((((~nextBlock)&0x7F7F7F7Fu)+0x7F7F7F7Fu)^0x80808080u); + if (escapeMask == 0) { + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + } else { + unsigned long firstEscapePos=__builtin_ctzl((unsigned long long) escapeMask)>>3; + switch(firstEscapePos) { /* Duff's device */ + case 3: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + // fall through + case 2: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + // fall through + case 1: code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + // fall through + case 0: posIn+=2; strOut[posOut++] = strIn[posIn-1]; /* decompress an escaped byte */ + } + } + } + if (posOut+32 <= size) { // handle the possibly 3 last bytes without a loop + if (posIn+2 <= lenIn) { + strOut[posOut] = strIn[posIn+1]; + if (strIn[posIn] != FSST_ESC) { + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + if (strIn[posIn] != FSST_ESC) { + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + } else { + posIn += 2; strOut[posOut++] = strIn[posIn-1]; + } + } else { + posIn += 2; posOut++; + } + } + if (posIn < lenIn) { // last code cannot be an escape + code = strIn[posIn++]; FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); posOut += len[code]; + } + } +#else + while (posOut+8 <= size && posIn < lenIn) + if ((code = strIn[posIn++]) < FSST_ESC) { /* symbol compressed as code? */ + FSST_UNALIGNED_STORE(strOut+posOut, symbol[code]); /* unaligned memory write */ + posOut += len[code]; + } else { + strOut[posOut] = strIn[posIn]; /* decompress an escaped byte */ + posIn++; posOut++; + } +#endif +#endif + while (posIn < lenIn) + if ((code = strIn[posIn++]) < FSST_ESC) { + size_t posWrite = posOut, endWrite = posOut + len[code]; + unsigned char* __restrict__ symbolPointer = ((unsigned char* __restrict__) &symbol[code]) - posWrite; + if ((posOut = endWrite) > size) endWrite = size; + for(; posWrite < endWrite; posWrite++) /* only write if there is room */ + strOut[posWrite] = symbolPointer[posWrite]; + } else { + if (posOut < size) strOut[posOut] = strIn[posIn]; /* idem */ + posIn++; posOut++; + } + if (posOut >= size && (decoder->zeroTerminated&1)) strOut[size-1] = 0; + return posOut; /* full size of decompressed string (could be >size, then the actually decompressed part) */ +} + +#ifdef __cplusplus +} +#endif +#endif /* FSST_INCLUDED_H */ diff --git a/cpp/thirdparty/fsst/fsst_avx512.cpp b/cpp/thirdparty/fsst/fsst_avx512.cpp new file mode 100644 index 000000000000..150683d2f9ac --- /dev/null +++ b/cpp/thirdparty/fsst/fsst_avx512.cpp @@ -0,0 +1,150 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +#include "libfsst.hpp" + +#if defined(__x86_64__) || defined(_M_X64) +#include + +#ifdef _WIN32 +namespace libfsst { +bool fsst_hasAVX512() { + int info[4]; + __cpuidex(info, 0x00000007, 0); + return (info[1]>>16)&1; +} +} // namespace libfsst +#else +#include +namespace libfsst { +bool fsst_hasAVX512() { + int info[4]; + __cpuid_count(0x00000007, 0, info[0], info[1], info[2], info[3]); + return (info[1]>>16)&1; +} +} // namespace libfsst +#endif +#else +namespace libfsst { +bool fsst_hasAVX512() { return false; } +} // namespace libfsst +#endif + +namespace libfsst { + +// BULK COMPRESSION OF STRINGS +// +// In one call of this function, we can compress 512 strings, each of maximum length 511 bytes. +// strings can be shorter than 511 bytes, no problem, but if they are longer we need to cut them up. +// +// In each iteration of the while loop, we find one code in each of the unroll*8 strings, i.e. (8,16,24 or 32) for resp. unroll=1,2,3,4 +// unroll3 performs best on my hardware +// +// In the worst case, each final encoded string occupies 512KB bytes (512*1024; with 1024=512xexception, exception = 2 bytes). +// - hence codeBase is a buffer of 512KB (needs 19 bits jobs), symbolBase of 256KB (needs 18 bits jobs). +// +// 'jobX' controls the encoding of each string and is therefore a u64 with format [out:19][pos:9][end:18][cur:18] (low-to-high bits) +// The field 'pos' tells which string we are processing (0..511). We need this info as strings will complete compressing out-of-order. +// +// Strings will have different lengths, and when a string is finished, we reload from the buffer of 512 input strings. +// This continues until we have less than (8,16,24 or 32; depending on unroll) strings left to process. +// - so 'processed' is the amount of strings we started processing and it is between [480,512]. +// Note that when we quit, there will still be some (<32) strings that we started to process but which are unfinished. +// - so 'unfinished' is that amount. These unfinished strings will be encoded further using the scalar method. +// +// Apart from the coded strings, we return in a output[] array of size 'processed' the job values of the 'finished' strings. +// In the following 'unfinished' slots (processed=finished+unfinished) we output the 'job' values of the unfinished strings. +// +// For the finished strings, we need [out:19] to see the compressed size and [pos:9] to see which string we refer to. +// For the unfinished strings, we need all fields of 'job' to continue the compression with scalar code (see SIMD code in compressBatch). +// +// THIS IS A SEPARATE CODE FILE NOT BECAUSE OF MY LOVE FOR MODULARIZED CODE BUT BECAUSE IT ALLOWS TO COMPILE IT WITH DIFFERENT FLAGS +// in particular, unrolling is crucial for gather/scatter performance, but requires registers. the #define all_* expressions however, +// will be detected to be constants by g++ -O2 and will be precomputed and placed into AVX512 registers - spoiling 9 of them. +// This reduces the effectiveness of unrolling, hence -O2 makes the loop perform worse than -O1 which skips this optimization. +// Assembly inspection confirmed that 3-way unroll with -O1 avoids needless load/stores. + +size_t fsst_compressAVX512(SymbolTable &symbolTable, u8* codeBase, u8* symbolBase, SIMDjob *input, SIMDjob *output, size_t n, size_t unroll) { + size_t processed = 0; + // define some constants (all_x means that all 8 lanes contain 64-bits value X) +#ifdef __AVX512F__ + //__m512i all_suffixLim= _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) symbolTable->suffixLim)); -- for variants b,c + __m512i all_MASK = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) -1)); + __m512i all_PRIME = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) FSST_HASH_PRIME)); + __m512i all_ICL_FREE = _mm512_broadcastq_epi64(_mm_set1_epi64((__m64) (u64) FSST_ICL_FREE)); +#define all_HASH _mm512_srli_epi64(all_MASK, 64-FSST_HASH_LOG2SIZE) +#define all_ONE _mm512_srli_epi64(all_MASK, 63) +#define all_M19 _mm512_srli_epi64(all_MASK, 45) +#define all_M18 _mm512_srli_epi64(all_MASK, 46) +#define all_M28 _mm512_srli_epi64(all_MASK, 36) +#define all_FFFFFF _mm512_srli_epi64(all_MASK, 40) +#define all_FFFF _mm512_srli_epi64(all_MASK, 48) +#define all_FF _mm512_srli_epi64(all_MASK, 56) + + SIMDjob *inputEnd = input+n; + assert(n >= unroll*8 && n <= 512); // should be close to 512 + __m512i job1, job2, job3, job4; // will contain current jobs, for each unroll 1,2,3,4 + __mmask8 loadmask1 = 255, loadmask2 = 255*(unroll>1), loadmask3 = 255*(unroll>2), loadmask4 = 255*(unroll>3); // 2b loaded new strings bitmask per unroll + u32 delta1 = 8, delta2 = 8*(unroll>1), delta3 = 8*(unroll>2), delta4 = 8*(unroll>3); // #new loads this SIMD iteration per unroll + + if (unroll >= 4) { + while (input+delta1+delta2+delta3+delta4 < inputEnd) { + #include "fsst_avx512_unroll4.inc" + } + } else if (unroll == 3) { + while (input+delta1+delta2+delta3 < inputEnd) { + #include "fsst_avx512_unroll3.inc" + } + } else if (unroll == 2) { + while (input+delta1+delta2 < inputEnd) { + #include "fsst_avx512_unroll2.inc" + } + } else { + while (input+delta1 < inputEnd) { + #include "fsst_avx512_unroll1.inc" + } + } + + // flush the job states of the unfinished strings at the end of output[] + processed = n - (inputEnd - input); + u32 unfinished = 0; + if (unroll > 1) { + if (unroll > 2) { + if (unroll > 3) { + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask4=~loadmask4, job4); + unfinished += _mm_popcnt_u32((int) loadmask4); + } + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask3=~loadmask3, job3); + unfinished += _mm_popcnt_u32((int) loadmask3); + } + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask2=~loadmask2, job2); + unfinished += _mm_popcnt_u32((int) loadmask2); + } + _mm512_mask_compressstoreu_epi64(output+unfinished, loadmask1=~loadmask1, job1); +#else + (void) symbolTable; + (void) codeBase; + (void) symbolBase; + (void) input; + (void) output; + (void) n; + (void) unroll; +#endif + return processed; +} +} // namespace libfsst + diff --git a/cpp/thirdparty/fsst/fsst_avx512_unroll1.inc b/cpp/thirdparty/fsst/fsst_avx512_unroll1.inc new file mode 100644 index 000000000000..f4b81c7970dd --- /dev/null +++ b/cpp/thirdparty/fsst/fsst_avx512_unroll1.inc @@ -0,0 +1,57 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; diff --git a/cpp/thirdparty/fsst/fsst_avx512_unroll2.inc b/cpp/thirdparty/fsst/fsst_avx512_unroll2.inc new file mode 100644 index 000000000000..aa33cd7e69c5 --- /dev/null +++ b/cpp/thirdparty/fsst/fsst_avx512_unroll2.inc @@ -0,0 +1,114 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + code2 = _mm512_and_epi64(code2, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + // calculate the amount of lanes in job2 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + delta2 = _mm_popcnt_u32((int) loadmask2); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; + _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; diff --git a/cpp/thirdparty/fsst/fsst_avx512_unroll3.inc b/cpp/thirdparty/fsst/fsst_avx512_unroll3.inc new file mode 100644 index 000000000000..e2057032abd3 --- /dev/null +++ b/cpp/thirdparty/fsst/fsst_avx512_unroll3.inc @@ -0,0 +1,171 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// +// +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E3PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// +// +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask3=11111111, delta3=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; + job3 = _mm512_mask_expandloadu_epi64(job3, loadmask3, input); input += delta3; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); + __m512i word3 = _mm512_i64gather_epi64(_mm512_srli_epi64(job3, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code3: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code3 = _mm512_i64gather_epi64(_mm512_and_epi64(word3, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + // get the first three bytes of the string. + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); + __m512i pos3 = _mm512_mullo_epi64(_mm512_and_epi64(word3, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT + // hash them into a random number: pos3 = pos3*PRIME; pos3 ^= pos3>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); + pos3 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos3,_mm512_srli_epi64(pos3,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write3 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); + __m512i write3 = _mm512_slli_epi64(_mm512_and_epi64(word3, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl3 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); + pos3 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl3, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); + __mmask8 match3 = _mm512_cmpeq_epi64_mask(symb3, _mm512_and_epi64(word3, pos3)) & _mm512_cmplt_epi64_mask(icl3, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); + code3 = _mm512_mask_mov_epi64(code3, match3, _mm512_srli_epi64(icl3, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); + write3 = _mm512_or_epi64(write3, _mm512_and_epi64(code3, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + code2 = _mm512_and_epi64(code2, all_FFFF); + code3 = _mm512_and_epi64(code3, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job3, all_M19), write3, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job3.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); + job3 = _mm512_add_epi64(job3, _mm512_slli_epi64(_mm512_srli_epi64(code3, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job3.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); + job3 = _mm512_add_epi64(job3, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code3, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) + // test which lanes are done now (job3.cur==job3.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job3 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); + loadmask3 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job3, 46), _mm512_and_epi64(_mm512_srli_epi64(job3, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + // calculate the amount of lanes in job2 that are done + // calculate the amount of lanes in job3 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + delta2 = _mm_popcnt_u32((int) loadmask2); + delta3 = _mm_popcnt_u32((int) loadmask3); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job3.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; + _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; + _mm512_mask_compressstoreu_epi64(output, loadmask3, job3); output += delta3; diff --git a/cpp/thirdparty/fsst/fsst_avx512_unroll4.inc b/cpp/thirdparty/fsst/fsst_avx512_unroll4.inc new file mode 100644 index 000000000000..15cca7c938b4 --- /dev/null +++ b/cpp/thirdparty/fsst/fsst_avx512_unroll4.inc @@ -0,0 +1,228 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// +// +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// +// +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// furnished to do so, subject to the following conditions: +// +// +// +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// +// +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E1PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E2PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E3PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, E4PRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// +// +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +// +// +// +// + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask1=11111111, delta1=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask2=11111111, delta2=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask3=11111111, delta3=8). + // load new jobs in the empty lanes (initially, all lanes are empty, so loadmask4=11111111, delta4=8). + job1 = _mm512_mask_expandloadu_epi64(job1, loadmask1, input); input += delta1; + job2 = _mm512_mask_expandloadu_epi64(job2, loadmask2, input); input += delta2; + job3 = _mm512_mask_expandloadu_epi64(job3, loadmask3, input); input += delta3; + job4 = _mm512_mask_expandloadu_epi64(job4, loadmask4, input); input += delta4; + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + // load the next 8 input string bytes (uncompressed data, aka 'symbols'). + __m512i word1 = _mm512_i64gather_epi64(_mm512_srli_epi64(job1, 46), symbolBase, 1); + __m512i word2 = _mm512_i64gather_epi64(_mm512_srli_epi64(job2, 46), symbolBase, 1); + __m512i word3 = _mm512_i64gather_epi64(_mm512_srli_epi64(job3, 46), symbolBase, 1); + __m512i word4 = _mm512_i64gather_epi64(_mm512_srli_epi64(job4, 46), symbolBase, 1); + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // load 16-bits codes from the 2-byte-prefix keyed lookup table. It also store 1-byte codes in all free slots. + // code1: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code2: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code3: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + // code4: Lowest 8 bits contain the code. Eleventh bit is whether it is an escaped code. Next 4 bits is length (2 or 1). + __m512i code1 = _mm512_i64gather_epi64(_mm512_and_epi64(word1, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code2 = _mm512_i64gather_epi64(_mm512_and_epi64(word2, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code3 = _mm512_i64gather_epi64(_mm512_and_epi64(word3, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + __m512i code4 = _mm512_i64gather_epi64(_mm512_and_epi64(word4, all_FFFF), symbolTable.shortCodes, sizeof(u16)); + // get the first three bytes of the string. + // get the first three bytes of the string. + // get the first three bytes of the string. + // get the first three bytes of the string. + __m512i pos1 = _mm512_mullo_epi64(_mm512_and_epi64(word1, all_FFFFFF), all_PRIME); + __m512i pos2 = _mm512_mullo_epi64(_mm512_and_epi64(word2, all_FFFFFF), all_PRIME); + __m512i pos3 = _mm512_mullo_epi64(_mm512_and_epi64(word3, all_FFFFFF), all_PRIME); + __m512i pos4 = _mm512_mullo_epi64(_mm512_and_epi64(word4, all_FFFFFF), all_PRIME); + // hash them into a random number: pos1 = pos1*PRIME; pos1 ^= pos1>>SHIFT + // hash them into a random number: pos2 = pos2*PRIME; pos2 ^= pos2>>SHIFT + // hash them into a random number: pos3 = pos3*PRIME; pos3 ^= pos3>>SHIFT + // hash them into a random number: pos4 = pos4*PRIME; pos4 ^= pos4>>SHIFT + pos1 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos1,_mm512_srli_epi64(pos1,FSST_SHIFT)), all_HASH), 4); + pos2 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos2,_mm512_srli_epi64(pos2,FSST_SHIFT)), all_HASH), 4); + pos3 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos3,_mm512_srli_epi64(pos3,FSST_SHIFT)), all_HASH), 4); + pos4 = _mm512_slli_epi64(_mm512_and_epi64(_mm512_xor_epi64(pos4,_mm512_srli_epi64(pos4,FSST_SHIFT)), all_HASH), 4); + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + // lookup in the 3-byte-prefix keyed hash table + __m512i icl1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 8), 1); + __m512i icl4 = _mm512_i64gather_epi64(pos4, (((char*) symbolTable.hashTab) + 8), 1); + // speculatively store the first input byte into the second position of the write1 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write2 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write3 register (in case it turns out to be an escaped byte). + // speculatively store the first input byte into the second position of the write4 register (in case it turns out to be an escaped byte). + __m512i write1 = _mm512_slli_epi64(_mm512_and_epi64(word1, all_FF), 8); + __m512i write2 = _mm512_slli_epi64(_mm512_and_epi64(word2, all_FF), 8); + __m512i write3 = _mm512_slli_epi64(_mm512_and_epi64(word3, all_FF), 8); + __m512i write4 = _mm512_slli_epi64(_mm512_and_epi64(word4, all_FF), 8); + // lookup just like the icl1 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl2 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl3 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + // lookup just like the icl4 above, but loads the next 8 bytes. This fetches the actual string bytes in the hash table. + __m512i symb1 = _mm512_i64gather_epi64(pos1, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb2 = _mm512_i64gather_epi64(pos2, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb3 = _mm512_i64gather_epi64(pos3, (((char*) symbolTable.hashTab) + 0), 1); + __m512i symb4 = _mm512_i64gather_epi64(pos4, (((char*) symbolTable.hashTab) + 0), 1); + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + // generate the FF..FF mask with an FF for each byte of the symbol (we need to AND the input with this to correctly check equality). + pos1 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl1, all_FF)); + pos2 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl2, all_FF)); + pos3 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl3, all_FF)); + pos4 = _mm512_srlv_epi64(all_MASK, _mm512_and_epi64(icl4, all_FF)); + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + // check symbol < |str| as well as whether it is an occupied slot (cmplt checks both conditions at once) and check string equality (cmpeq). + __mmask8 match1 = _mm512_cmpeq_epi64_mask(symb1, _mm512_and_epi64(word1, pos1)) & _mm512_cmplt_epi64_mask(icl1, all_ICL_FREE); + __mmask8 match2 = _mm512_cmpeq_epi64_mask(symb2, _mm512_and_epi64(word2, pos2)) & _mm512_cmplt_epi64_mask(icl2, all_ICL_FREE); + __mmask8 match3 = _mm512_cmpeq_epi64_mask(symb3, _mm512_and_epi64(word3, pos3)) & _mm512_cmplt_epi64_mask(icl3, all_ICL_FREE); + __mmask8 match4 = _mm512_cmpeq_epi64_mask(symb4, _mm512_and_epi64(word4, pos4)) & _mm512_cmplt_epi64_mask(icl4, all_ICL_FREE); + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + // for the hits, overwrite the codes with what comes from the hash table (codes for symbols of length >=3). The rest stays with what shortCodes gave. + code1 = _mm512_mask_mov_epi64(code1, match1, _mm512_srli_epi64(icl1, 16)); + code2 = _mm512_mask_mov_epi64(code2, match2, _mm512_srli_epi64(icl2, 16)); + code3 = _mm512_mask_mov_epi64(code3, match3, _mm512_srli_epi64(icl3, 16)); + code4 = _mm512_mask_mov_epi64(code4, match4, _mm512_srli_epi64(icl4, 16)); + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + // write out the code byte as the first output byte. Notice that this byte may also be the escape code 255 (for escapes) coming from shortCodes. + write1 = _mm512_or_epi64(write1, _mm512_and_epi64(code1, all_FF)); + write2 = _mm512_or_epi64(write2, _mm512_and_epi64(code2, all_FF)); + write3 = _mm512_or_epi64(write3, _mm512_and_epi64(code3, all_FF)); + write4 = _mm512_or_epi64(write4, _mm512_and_epi64(code4, all_FF)); + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + // zip the irrelevant 6 bytes (just stay with the 2 relevant bytes containing the 16-bits code) + code1 = _mm512_and_epi64(code1, all_FFFF); + code2 = _mm512_and_epi64(code2, all_FFFF); + code3 = _mm512_and_epi64(code3, all_FFFF); + code4 = _mm512_and_epi64(code4, all_FFFF); + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + // write out the compressed data. It writes 8 bytes, but only 1 byte is relevant :-(or 2 bytes are, in case of an escape code) + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job1, all_M19), write1, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job2, all_M19), write2, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job3, all_M19), write3, 1); + _mm512_i64scatter_epi64(codeBase, _mm512_and_epi64(job4, all_M19), write4, 1); + // increase the job1.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job2.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job3.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + // increase the job4.cur field in the job with the symbol length (for this, shift away 12 bits from the code) + job1 = _mm512_add_epi64(job1, _mm512_slli_epi64(_mm512_srli_epi64(code1, FSST_LEN_BITS), 46)); + job2 = _mm512_add_epi64(job2, _mm512_slli_epi64(_mm512_srli_epi64(code2, FSST_LEN_BITS), 46)); + job3 = _mm512_add_epi64(job3, _mm512_slli_epi64(_mm512_srli_epi64(code3, FSST_LEN_BITS), 46)); + job4 = _mm512_add_epi64(job4, _mm512_slli_epi64(_mm512_srli_epi64(code4, FSST_LEN_BITS), 46)); + // increase the job1.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job2.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job3.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + // increase the job4.out' field with one, or two in case of an escape code (add 1 plus the escape bit, i.e the 8th) + job1 = _mm512_add_epi64(job1, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code1, 8), all_ONE))); + job2 = _mm512_add_epi64(job2, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code2, 8), all_ONE))); + job3 = _mm512_add_epi64(job3, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code3, 8), all_ONE))); + job4 = _mm512_add_epi64(job4, _mm512_add_epi64(all_ONE, _mm512_and_epi64(_mm512_srli_epi64(code4, 8), all_ONE))); + // test which lanes are done now (job1.cur==job1.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job1 register) + // test which lanes are done now (job2.cur==job2.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job2 register) + // test which lanes are done now (job3.cur==job3.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job3 register) + // test which lanes are done now (job4.cur==job4.end), cur starts at bit 46, end starts at bit 28 (the highest 2x18 bits in the job4 register) + loadmask1 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job1, 46), _mm512_and_epi64(_mm512_srli_epi64(job1, 28), all_M18)); + loadmask2 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job2, 46), _mm512_and_epi64(_mm512_srli_epi64(job2, 28), all_M18)); + loadmask3 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job3, 46), _mm512_and_epi64(_mm512_srli_epi64(job3, 28), all_M18)); + loadmask4 = _mm512_cmpeq_epi64_mask(_mm512_srli_epi64(job4, 46), _mm512_and_epi64(_mm512_srli_epi64(job4, 28), all_M18)); + // calculate the amount of lanes in job1 that are done + // calculate the amount of lanes in job2 that are done + // calculate the amount of lanes in job3 that are done + // calculate the amount of lanes in job4 that are done + delta1 = _mm_popcnt_u32((int) loadmask1); + delta2 = _mm_popcnt_u32((int) loadmask2); + delta3 = _mm_popcnt_u32((int) loadmask3); + delta4 = _mm_popcnt_u32((int) loadmask4); + // write out the job state for the lanes that are done (we need the final 'job1.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job2.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job3.out' value to compute the compressed string length) + // write out the job state for the lanes that are done (we need the final 'job4.out' value to compute the compressed string length) + _mm512_mask_compressstoreu_epi64(output, loadmask1, job1); output += delta1; + _mm512_mask_compressstoreu_epi64(output, loadmask2, job2); output += delta2; + _mm512_mask_compressstoreu_epi64(output, loadmask3, job3); output += delta3; + _mm512_mask_compressstoreu_epi64(output, loadmask4, job4); output += delta4; diff --git a/cpp/thirdparty/fsst/libfsst.cpp b/cpp/thirdparty/fsst/libfsst.cpp new file mode 100644 index 000000000000..e3ba787b9592 --- /dev/null +++ b/cpp/thirdparty/fsst/libfsst.cpp @@ -0,0 +1,651 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +#include "libfsst.hpp" + +namespace libfsst { +Symbol concat(Symbol a, Symbol b) { + Symbol s; + u32 length = a.length()+b.length(); + if (length > Symbol::maxLength) length = Symbol::maxLength; + s.set_code_len(FSST_CODE_MASK, length); + s.store_num((b.load_num() << (8*a.length())) | a.load_num()); + return s; +} +} // namespace libfsst + +namespace std { +template <> +class hash { + public: + size_t operator()(const libfsst::QSymbol& q) const { + uint64_t k = q.symbol.load_num(); + const uint64_t m = 0xc6a4a7935bd1e995; + const int r = 47; + uint64_t h = 0x8445d61a4e774912 ^ (8*m); + k *= m; + k ^= k >> r; + k *= m; + h ^= k; + h *= m; + h ^= h >> r; + h *= m; + h ^= h >> r; + return h; + } +}; +} + +namespace libfsst { +bool isEscapeCode(u16 pos) { return pos < FSST_CODE_BASE; } + +std::ostream& operator<<(std::ostream& out, const Symbol& s) { + for (u32 i=0; i line, const size_t len[], bool zeroTerminated=false) { + SymbolTable *st = new SymbolTable(), *bestTable = new SymbolTable(); + int bestGain = (int) -FSST_SAMPLEMAXSZ; // worst case (everything exception) + size_t sampleFrac = 128; + + // start by determining the terminator. We use the (lowest) most infrequent byte as terminator + st->zeroTerminated = zeroTerminated; + if (zeroTerminated) { + st->terminator = 0; // except in case of zeroTerminated mode, then byte 0 is terminator regardless frequency + } else { + u16 byteHisto[256]; + memset(byteHisto, 0, sizeof(byteHisto)); + for(size_t i=0; iterminator = 256; + while(i-- > 0) { + if (byteHisto[i] > minSize) continue; + st->terminator = i; + minSize = byteHisto[i]; + } + } + assert(st->terminator != 256); + + // a random number between 0 and 128 + auto rnd128 = [&](size_t i) { return 1 + (FSST_HASH((i+1UL)*sampleFrac)&127); }; + + // compress sample, and compute (pair-)frequencies + auto compressCount = [&](SymbolTable *st, Counters &counters) { // returns gain + int gain = 0; + + for(size_t i=0; i sampleFrac) continue; + } + if (cur < end) { + u16 code2 = 255, code1 = st->findLongestSymbol(cur, end); + cur += st->symbols[code1].length(); + gain += (int) (st->symbols[code1].length()-(1+isEscapeCode(code1))); + while (true) { + // count single symbol (i.e. an option is not extending it) + counters.count1Inc(code1); + + // as an alternative, consider just using the next byte.. + if (st->symbols[code1].length() != 1) // .. but do not count single byte symbols doubly + counters.count1Inc(*start); + + if (cur==end) { + break; + } + + // now match a new symbol + start = cur; + if (curhashTabSize-1); + Symbol s = st->hashTab[idx]; + code2 = st->shortCodes[word & 0xFFFF] & FSST_CODE_MASK; + word &= (0xFFFFFFFFFFFFFFFF >> (u8) s.icl); + if ((s.icl < FSST_ICL_FREE) & (s.load_num() == word)) { + code2 = s.code(); + cur += s.length(); + } else if (code2 >= FSST_CODE_BASE) { + cur += 2; + } else { + code2 = st->byteCodes[word & 0xFF] & FSST_CODE_MASK; + cur += 1; + } + } else { + code2 = st->findLongestSymbol(cur, end); + cur += st->symbols[code2].length(); + } + + // compute compressed output size + gain += ((int) (cur-start))-(1+isEscapeCode(code2)); + + if (sampleFrac < 128) { // no need to count pairs in final round + // consider the symbol that is the concatenation of the two last symbols + counters.count2Inc(code1, code2); + + // as an alternative, consider just extending with the next byte.. + if ((cur-start) > 1) // ..but do not count single byte extensions doubly + counters.count2Inc(code1, *start); + } + code1 = code2; + } + } + } + return gain; + }; + + auto makeTable = [&](SymbolTable *st, Counters &counters) { + // hashmap of c (needed because we can generate duplicate candidates) + unordered_set cands; + + // artificially make terminater the most frequent symbol so it gets included + u16 terminator = st->nSymbols?FSST_CODE_BASE:st->terminator; + counters.count1Set(terminator,65535); + + auto addOrInc = [&](unordered_set &cands, Symbol s, u64 count) { + if (count < (5*sampleFrac)/128) return; // improves both compression speed (less candidates), but also quality!! + QSymbol q; + q.symbol = s; + q.gain = count * s.length(); + auto it = cands.find(q); + if (it != cands.end()) { + q.gain += (*it).gain; + cands.erase(*it); + } + cands.insert(q); + }; + + // add candidate symbols based on counted frequency + for (u32 pos1=0; pos1nSymbols; pos1++) { + u32 cnt1 = counters.count1GetNext(pos1); // may advance pos1!! + if (!cnt1) continue; + + // heuristic: promoting single-byte symbols (*8) helps reduce exception rates and increases [de]compression speed + Symbol s1 = st->symbols[pos1]; + addOrInc(cands, s1, ((s1.length()==1)?8LL:1LL)*cnt1); + + if (sampleFrac >= 128 || // last round we do not create new (combined) symbols + s1.length() == Symbol::maxLength || // symbol cannot be extended + s1.val.str[0] == st->terminator) { // multi-byte symbols cannot contain the terminator byte + continue; + } + for (u32 pos2=0; pos2nSymbols; pos2++) { + u32 cnt2 = counters.count2GetNext(pos1, pos2); // may advance pos2!! + if (!cnt2) continue; + + // create a new symbol + Symbol s2 = st->symbols[pos2]; + Symbol s3 = concat(s1, s2); + if (s2.val.str[0] != st->terminator) // multi-byte symbols cannot contain the terminator byte + addOrInc(cands, s3, cnt2); + } + } + + // insert candidates into priority queue (by gain) + auto cmpGn = [](const QSymbol& q1, const QSymbol& q2) { return (q1.gain < q2.gain) || (q1.gain == q2.gain && q1.symbol.load_num() > q2.symbol.load_num()); }; + priority_queue,decltype(cmpGn)> pq(cmpGn); + for (auto& q : cands) + pq.push(q); + + // Create new symbol map using best candidates + st->clear(); + while (st->nSymbols < 255 && !pq.empty()) { + QSymbol q = pq.top(); + pq.pop(); + st->add(q.symbol); + } + }; + + u8 bestCounters[512*sizeof(u16)]; +#ifdef NONOPT_FSST + for(size_t frac : {127, 127, 127, 127, 127, 127, 127, 127, 127, 128}) { + sampleFrac = frac; +#else + for(sampleFrac=8; true; sampleFrac += 30) { +#endif + memset(&counters, 0, sizeof(Counters)); + long gain = compressCount(st, counters); + if (gain >= bestGain) { // a new best solution! + counters.backup1(bestCounters); + *bestTable = *st; bestGain = gain; + } + if (sampleFrac >= 128) break; // we do 5 rounds (sampleFrac=8,38,68,98,128) + makeTable(st, counters); + } + delete st; + counters.restore1(bestCounters); + makeTable(bestTable, counters); + bestTable->finalize(zeroTerminated); // renumber codes for more efficient compression + return bestTable; +} + +#ifndef NONOPT_FSST +static inline size_t compressSIMD(SymbolTable &symbolTable, u8* symbolBase, size_t nlines, const size_t len[], const u8* line[], size_t size, u8* dst, size_t lenOut[], u8* strOut[], int unroll) { + size_t curLine = 0, inOff = 0, outOff = 0, batchPos = 0, empty = 0, budget = size; + u8 *lim = dst + size, *codeBase = symbolBase + (1<<18); // 512KB temp space for compressing 512 strings + SIMDjob input[512]; // combined offsets of input strings (cur,end), and string #id (pos) and output (dst) pointer + SIMDjob output[512]; // output are (pos:9,dst:19) end pointers (compute compressed length from this) + size_t jobLine[512]; // for which line in the input sequence was this job (needed because we may split a line into multiple jobs) + + while (curLine < nlines && outOff <= (1<<19)) { + size_t prevLine = curLine, chunk, curOff = 0; + + // bail out if the output buffer cannot hold the compressed next string fully + if (((len[curLine]-curOff)*2 + 7) > budget) break; // see below for the +7 + else budget -= (len[curLine]-curOff)*2; + + strOut[curLine] = (u8*) 0; + lenOut[curLine] = 0; + + do { + do { + chunk = len[curLine] - curOff; + if (chunk > 511) { + chunk = 511; // large strings need to be chopped up into segments of 511 bytes + } + // create a job in this batch + SIMDjob job; + job.cur = inOff; + job.end = job.cur + chunk; + job.pos = batchPos; + job.out = outOff; + + // worst case estimate for compressed size (+7 is for the scatter that writes extra 7 zeros) + outOff += 7 + 2*(size_t)(job.end - job.cur); // note, total size needed is 512*(511*2+7) bytes. + if (outOff > (1<<19)) break; // simdbuf may get full, stop before this chunk + + // register job in this batch + input[batchPos] = job; + jobLine[batchPos] = curLine; + + if (chunk == 0) { + empty++; // detect empty chunks -- SIMD code cannot handle empty strings, so they need to be filtered out + } else { + // copy string chunk into temp buffer + memcpy(symbolBase + inOff, line[curLine] + curOff, chunk); + inOff += chunk; + curOff += chunk; + symbolBase[inOff++] = (u8) symbolTable.terminator; // write an extra char at the end that will not be encoded + } + if (++batchPos == 512) break; + } while(curOff < len[curLine]); + + if ((batchPos == 512) || (outOff > (1<<19)) || (++curLine >= nlines) || (((len[curLine])*2 + 7) > budget)) { // cannot accumulate more? + if (batchPos-empty >= 32) { // if we have enough work, fire off fsst_compressAVX512 (32 is due to max 4x8 unrolling) + // radix-sort jobs on length (longest string first) + // -- this provides best load balancing and allows to skip empty jobs at the end + u16 sortpos[513]; + memset(sortpos, 0, sizeof(sortpos)); + + // calculate length histo + for(size_t i=0; i> (u8) s.icl); + if ((s.icl < FSST_ICL_FREE) && s.load_num() == word) { + *out++ = (u8) s.code(); cur += s.length(); + } else { + // could be a 2-byte or 1-byte code, or miss + // handle everything with predication + *out = (u8) code; + out += 1+((code&FSST_CODE_BASE)>>8); + cur += (code>>FSST_LEN_BITS); + } + } + job.out = out - codeBase; + } + // postprocess job info + job.cur = 0; + job.end = job.out - input[job.pos].out; // misuse .end field as compressed size + job.out = input[job.pos].out; // reset offset to start of encoded string + input[job.pos] = job; + } + + // copy out the result data + for(size_t i=0; i> (u8) s.icl); + if ((s.icl < FSST_ICL_FREE) && s.load_num() == word) { + *out++ = (u8) s.code(); cur += s.length(); + } else if (avoidBranch) { + // could be a 2-byte or 1-byte code, or miss + // handle everything with predication + *out = (u8) code; + out += 1+((code&FSST_CODE_BASE)>>8); + cur += (code>>FSST_LEN_BITS); + } else if ((u8) code < byteLim) { + // 2 byte code after checking there is no longer pattern + *out++ = (u8) code; cur += 2; + } else { + // 1 byte code or miss. + *out = (u8) code; + out += 1+((code&FSST_CODE_BASE)>>8); // predicated - tested with a branch, that was always worse + cur++; + } + } + } + }; + + for(curLine=0; curLine 511) { + chunk = 511; // we need to compress in chunks of 511 in order to be byte-compatible with simd-compressed FSST + } + if ((2*chunk+7) > (size_t) (lim-out)) { + return curLine; // out of memory + } + // copy the string to the 511-byte buffer + memcpy(buf, cur, chunk); + buf[chunk] = (u8) symbolTable.terminator; + cur = buf; + end = cur + chunk; + + // based on symboltable stats, choose a variant that is nice to the branch predictor + if (noSuffixOpt) { + compressVariant(true,false); + } else if (avoidBranch) { + compressVariant(false,true); + } else { + compressVariant(false, false); + } + } while((curOff += chunk) < lenIn[curLine]); + lenOut[curLine] = (size_t) (out - strOut[curLine]); + } + return curLine; +} + +#define FSST_SAMPLELINE ((size_t) 512) + +// quickly select a uniformly random set of lines such that we have between [FSST_SAMPLETARGET,FSST_SAMPLEMAXSZ) string bytes +vector makeSample(u8* sampleBuf, const u8* strIn[], const size_t **lenRef, size_t nlines) { + size_t totSize = 0; + const size_t *lenIn = *lenRef; + vector sample; + + for(size_t i=0; i sample = makeSample(sampleBuf, strIn, &sampleLen, n?n:1); // careful handling of input to get a right-size and representative sample + Encoder *encoder = new Encoder(); + encoder->symbolTable = shared_ptr(buildSymbolTable(encoder->counters, sample, sampleLen, zeroTerminated)); + if (sampleLen != lenIn) delete[] sampleLen; + delete[] sampleBuf; + return (fsst_encoder_t*) encoder; +} + +/* create another encoder instance, necessary to do multi-threaded encoding using the same symbol table */ +extern "C" fsst_encoder_t* fsst_duplicate(fsst_encoder_t *encoder) { + Encoder *e = new Encoder(); + e->symbolTable = ((Encoder*)encoder)->symbolTable; // it is a shared_ptr + return (fsst_encoder_t*) e; +} + +// export a symbol table in compact format. +extern "C" u32 fsst_export(fsst_encoder_t *encoder, u8 *buf) { + Encoder *e = (Encoder*) encoder; + // In ->version there is a versionnr, but we hide also suffixLim/terminator/nSymbols there. + // This is sufficient in principle to *reconstruct* a fsst_encoder_t from a fsst_decoder_t + // (such functionality could be useful to append compressed data to an existing block). + // + // However, the hash function in the encoder hash table is endian-sensitive, and given its + // 'lossy perfect' hashing scheme is *unable* to contain other-endian-produced symbol tables. + // Doing a endian-conversion during hashing will be slow and self-defeating. + // + // Overall, we could support reconstructing an encoder for incremental compression, but + // should enforce equal-endianness. Bit of a bummer. Not going there now. + // + // The version field is now there just for future-proofness, but not used yet + + // version allows keeping track of fsst versions, track endianness, and encoder reconstruction + u64 version = (FSST_VERSION << 32) | // version is 24 bits, most significant byte is 0 + (((u64) e->symbolTable->suffixLim) << 24) | + (((u64) e->symbolTable->terminator) << 16) | + (((u64) e->symbolTable->nSymbols) << 8) | + FSST_ENDIAN_MARKER; // least significant byte is nonzero + + version = swap64_if_be(version); // ensure version is little-endian encoded + + /* do not assume unaligned reads here */ + memcpy(buf, &version, 8); + buf[8] = e->symbolTable->zeroTerminated; + for(u32 i=0; i<8; i++) + buf[9+i] = (u8) e->symbolTable->lenHisto[i]; + u32 pos = 17; + + // emit only the used bytes of the symbols + for(u32 i = e->symbolTable->zeroTerminated; i < e->symbolTable->nSymbols; i++) + for(u32 j = 0; j < e->symbolTable->symbols[i].length(); j++) + buf[pos++] = e->symbolTable->symbols[i].val.str[j]; // serialize used symbol bytes + + return pos; // length of what was serialized +} + +#define FSST_CORRUPT 32774747032022883 /* 7-byte number in little endian containing "corrupt" */ + +extern "C" u32 fsst_import(fsst_decoder_t *decoder, u8 const *buf) { + u64 version = 0; + u32 code, pos = 17; + u8 lenHisto[8]; + + // version field (first 8 bytes) is now there just for future-proofness, unused still (skipped) + memcpy(&version, buf, 8); + version = swap64_if_be(version); // version is always little-endian encoded + + if ((version>>32) != FSST_VERSION) return 0; + decoder->zeroTerminated = buf[8]&1; + memcpy(lenHisto, buf+9, 8); + + // in case of zero-terminated, first symbol is "" (zero always, may be overwritten) + decoder->len[0] = 1; + decoder->symbol[0] = 0; + + // we use lenHisto[0] as 1-byte symbol run length (at the end) + code = decoder->zeroTerminated; + if (decoder->zeroTerminated) lenHisto[0]--; // if zeroTerminated, then symbol "" aka 1-byte code=0, is not stored at the end + + // now get all symbols from the buffer + for(u32 l=1; l<=8; l++) { /* l = 1,2,3,4,5,6,7,8 */ + for(u32 i=0; i < lenHisto[(l&7) /* 1,2,3,4,5,6,7,0 */]; i++, code++) { + decoder->len[code] = (l&7)+1; /* len = 2,3,4,5,6,7,8,1 */ + decoder->symbol[code] = 0; + for(u32 j=0; jlen[code]; j++) + ((u8*) &decoder->symbol[code])[j] = buf[pos++]; // note this enforces 'little endian' symbols + } + } + if (decoder->zeroTerminated) lenHisto[0]++; + + // fill unused symbols with text "corrupt". Gives a chance to detect corrupted code sequences (if there are unused symbols). + while(code<255) { + decoder->symbol[code] = FSST_CORRUPT; + decoder->len[code++] = 8; + } + return pos; +} + +// runtime check for simd +inline size_t _compressImpl(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd) { +#ifndef NONOPT_FSST + if (simd && fsst_hasAVX512()) + return compressSIMD(*e->symbolTable, e->simdbuf, nlines, lenIn, strIn, size, output, lenOut, strOut, simd); +#endif + (void) simd; + return compressBulk(*e->symbolTable, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch); +} +size_t compressImpl(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd) { + return _compressImpl(e, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch, simd); +} + +// adaptive choosing of scalar compression method based on symbol length histogram +inline size_t _compressAuto(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], int simd) { + bool avoidBranch = false, noSuffixOpt = false; + if (100*e->symbolTable->lenHisto[1] > 65*e->symbolTable->nSymbols && 100*e->symbolTable->suffixLim > 95*e->symbolTable->lenHisto[1]) { + noSuffixOpt = true; + } else if ((e->symbolTable->lenHisto[0] > 24 && e->symbolTable->lenHisto[0] < 92) && + (e->symbolTable->lenHisto[0] < 43 || e->symbolTable->lenHisto[6] + e->symbolTable->lenHisto[7] < 29) && + (e->symbolTable->lenHisto[0] < 72 || e->symbolTable->lenHisto[2] < 72)) { + avoidBranch = true; + } + return _compressImpl(e, nlines, lenIn, strIn, size, output, lenOut, strOut, noSuffixOpt, avoidBranch, simd); +} +size_t compressAuto(Encoder *e, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[], int simd) { + return _compressAuto(e, nlines, lenIn, strIn, size, output, lenOut, strOut, simd); +} +} // namespace libfsst + +using namespace libfsst; +// the main compression function (everything automatic) +extern "C" size_t fsst_compress(fsst_encoder_t *encoder, size_t nlines, const size_t lenIn[], const u8 *strIn[], size_t size, u8 *output, size_t *lenOut, u8 *strOut[]) { + // to be faster than scalar, simd needs 64 lines or more of length >=12; or fewer lines, but big ones (totLen > 32KB) + size_t totLen = accumulate(lenIn, lenIn+nlines, 0); + int simd = totLen > nlines*12 && (nlines > 64 || totLen > (size_t) 1<<15); + return _compressAuto((Encoder*) encoder, nlines, lenIn, strIn, size, output, lenOut, strOut, 3*simd); +} + +/* deallocate encoder */ +extern "C" void fsst_destroy(fsst_encoder_t* encoder) { + Encoder *e = (Encoder*) encoder; + delete e; +} + +/* very lazy implementation relying on export and import */ +extern "C" fsst_decoder_t fsst_decoder(fsst_encoder_t *encoder) { + u8 buf[sizeof(fsst_decoder_t)]; + u32 cnt1 = fsst_export(encoder, buf); + fsst_decoder_t decoder; + u32 cnt2 = fsst_import(&decoder, buf); + assert(cnt1 == cnt2); (void) cnt1; (void) cnt2; + return decoder; +} diff --git a/cpp/thirdparty/fsst/libfsst.hpp b/cpp/thirdparty/fsst/libfsst.hpp new file mode 100644 index 000000000000..f61bc0175b63 --- /dev/null +++ b/cpp/thirdparty/fsst/libfsst.hpp @@ -0,0 +1,471 @@ +// this software is distributed under the MIT License (http://www.opensource.org/licenses/MIT): +// +// Copyright 2018-2020, CWI, TU Munich, FSU Jena +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files +// (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, +// merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// +// You can contact the authors via the FSST source repository : https://github.com/cwida/fsst +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +#include "fsst.h" // the official FSST API -- also usable by C mortals + +/* unsigned integers */ +namespace libfsst { +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +} // namespace libfsst + +#if UINTPTR_MAX == 0xffffffffU +// We're on a 32-bit platform +#define NONOPT_FSST +#endif + +#define FSST_ENDIAN_MARKER ((u64) 1) +#define FSST_VERSION_20190218 20190218 +#define FSST_VERSION ((u64) FSST_VERSION_20190218) + +// "symbols" are character sequences (up to 8 bytes) +// A symbol is compressed into a "code" of, in principle, one byte. But, we added an exception mechanism: +// byte 255 followed by byte X represents the single-byte symbol X. Its code is 256+X. + +// we represent codes in u16 (not u8). 12 bits code (of which 10 are used), 4 bits length +#define FSST_LEN_BITS 12 +#define FSST_CODE_BITS 9 +#define FSST_CODE_BASE 256UL /* first 256 codes [0,255] are pseudo codes: escaped bytes */ +#define FSST_CODE_MAX (1UL<=8) { + len = 8; + memcpy(val.str, input, 8); + } else { + memcpy(val.str, input, len); + } + set_code_len(FSST_CODE_MAX, len); + } + void set_code_len(u32 code, u32 len) { icl = (len<<28)|(code<<16)|((8-len)*8); } + + u64 load_num() const { return swap64_if_be(val.num); } + void store_num(u64 v) { val.num = swap64_if_be(v); } + + u32 length() const { return (u32) (icl >> 28); } + u16 code() const { return (icl >> 16) & FSST_CODE_MASK; } + u32 ignoredBits() const { return (u32) icl; } + + u8 first() const { assert( length() >= 1); return 0xFF & load_num(); } + u16 first2() const { assert( length() >= 2); return 0xFFFF & load_num(); } + +#define FSST_HASH_LOG2SIZE 10 +#define FSST_HASH_PRIME 2971215073LL +#define FSST_SHIFT 15 +#define FSST_HASH(w) (((w)*FSST_HASH_PRIME)^(((w)*FSST_HASH_PRIME)>>FSST_SHIFT)) + size_t hash() const { size_t v = 0xFFFFFF & load_num(); return FSST_HASH(v); } // hash on the next 3 bytes +}; + +// Symbol that can be put in a queue, ordered on gain +struct QSymbol{ + Symbol symbol; + mutable u32 gain; // mutable because gain value should be ignored in find() on unordered_set of QSymbols + bool operator==(const QSymbol& other) const { return symbol.val.num == other.symbol.val.num && symbol.length() == other.symbol.length(); } +}; + +// we construct FSST symbol tables using a random sample of about 16KB (1<<14) +#define FSST_SAMPLETARGET (1<<14) +#define FSST_SAMPLEMAXSZ ((long) 2*FSST_SAMPLETARGET) + +// two phases of compression, before and after optimize(): +// +// (1) to encode values we probe (and maintain) three datastructures: +// - u16 byteCodes[256] array at the position of the next byte (s.length==1) +// - u16 shortCodes[65536] array at the position of the next twobyte pattern (s.length==2) +// - Symbol hashtable[1024] (keyed by the next three bytes, ie for s.length>2), +// this search will yield a u16 code, it points into Symbol symbols[]. You always find a hit, because the first 256 codes are +// pseudo codes representing a single byte these will become escapes) +// +// (2) when we finished looking for the best symbol table we call optimize() to reshape it: +// - it renumbers the codes by length (first symbols of length 2,3,4,5,6,7,8; then 1 (starting from byteLim are symbols of length 1) +// length 2 codes for which no longer suffix symbol exists (< suffixLim) come first among the 2-byte codes +// (allows shortcut during compression) +// - for each two-byte combination, in all unused slots of shortCodes[], it enters the byteCode[] of the symbol corresponding +// to the first byte (if such a single-byte symbol exists). This allows us to just probe the next two bytes (if there is only one +// byte left in the string, there is still a terminator-byte added during compression) in shortCodes[]. That is, byteCodes[] +// and its codepath is no longer required. This makes compression faster. The reason we use byteCodes[] during symbolTable construction +// is that adding a new code/symbol is expensive (you have to touch shortCodes[] in 256 places). This optimization was +// hence added to make symbolTable construction faster. +// +// this final layout allows for the fastest compression code, only currently present in compressBulk + +// in the hash table, the icl field contains (low-to-high) ignoredBits:16,code:12,length:4 +#define FSST_ICL_FREE ((15<<28)|(((u32)FSST_CODE_MASK)<<16)) // high bits of icl (len=8,code=FSST_CODE_MASK) indicates free bucket + +// ignoredBits is (8-length)*8, which is the amount of high bits to zero in the input word before comparing with the hashtable key +// ..it could of course be computed from len during lookup, but storing it precomputed in some loose bits is faster +// +// the gain field is only used in the symbol queue that sorts symbols on gain + +struct SymbolTable { + static const u32 hashTabSize = 1<> (u8) s.icl)); + return true; + } + bool add(Symbol s) { + assert(FSST_CODE_BASE + nSymbols < FSST_CODE_MAX); + u32 len = s.length(); + s.set_code_len(FSST_CODE_BASE + nSymbols, len); + if (len == 1) { + byteCodes[s.first()] = FSST_CODE_BASE + nSymbols + (1<> ((u8) hashTab[idx].icl)))) { + return (hashTab[idx].icl>>16) & FSST_CODE_MASK; // matched a long symbol + } + if (s.length() >= 2) { + u16 code = shortCodes[s.first2()] & FSST_CODE_MASK; + if (code >= FSST_CODE_BASE) return code; + } + return byteCodes[s.first()] & FSST_CODE_MASK; + } + u16 findLongestSymbol(const u8* cur, const u8* end) const { + return findLongestSymbol(Symbol(cur,end)); // represent the string as a temporary symbol + } + + // rationale for finalize: + // - during symbol table construction, we may create more than 256 codes, but bring it down to max 255 in the last makeTable() + // consequently we needed more than 8 bits during symbol table contruction, but can simplify the codes to single bytes in finalize() + // (this feature is in fact lo longer used, but could still be exploited: symbol construction creates no more than 255 symbols in each pass) + // - we not only reduce the amount of codes to <255, but also *reorder* the symbols and renumber their codes, for higher compression perf. + // we renumber codes so they are grouped by length, to allow optimized scalar string compression (byteLim and suffixLim optimizations). + // - we make the use of byteCode[] no longer necessary by inserting single-byte codes in the free spots of shortCodes[] + // Using shortCodes[] only makes compression faster. When creating the symbolTable, however, using shortCodes[] for the single-byte + // symbols is slow, as each insert touches 256 positions in it. This optimization was added when optimizing symbolTable construction time. + // + // In all, we change the layout and coding, as follows.. + // + // before finalize(): + // - The real symbols are symbols[256..256+nSymbols>. As we may have nSymbols > 255 + // - The first 256 codes are pseudo symbols (all escaped bytes) + // + // after finalize(): + // - table layout is symbols[0..nSymbols>, with nSymbols < 256. + // - Real codes are [0,nSymbols>. 8-th bit not set. + // - Escapes in shortCodes have the 8th bit set (value: 256+255=511). 255 because the code to be emitted is the escape byte 255 + // - symbols are grouped by length: 2,3,4,5,6,7,8, then 1 (single-byte codes last) + // the two-byte codes are split in two sections: + // - first section contains codes for symbols for which there is no longer symbol (no suffix). It allows an early-out during compression + // + // finally, shortCodes[] is modified to also encode all single-byte symbols (hence byteCodes[] is not required on a critical path anymore). + // + void finalize(u8 zeroTerminated) { + assert(nSymbols <= 255); + u8 newCode[256], rsum[8], byteLim = nSymbols - (lenHisto[0] - zeroTerminated); + + // compute running sum of code lengths (starting offsets for each length) + rsum[0] = byteLim; // 1-byte codes are highest + rsum[1] = zeroTerminated; + for(u32 i=1; i<7; i++) + rsum[i+1] = rsum[i] + lenHisto[i]; + + // determine the new code for each symbol, ordered by length (and splitting 2byte symbols into two classes around suffixLim) + suffixLim = rsum[1]; + symbols[newCode[0] = 0] = symbols[256]; // keep symbol 0 in place (for zeroTerminated cases only) + + for(u32 i=zeroTerminated, j=rsum[2]; i 1 && first2 == s2.first2()) // test if symbol k is a suffix of s + opt = 0; + } + newCode[i] = opt?suffixLim++:--j; // symbols without a larger suffix have a code < suffixLim + } else + newCode[i] = rsum[len-1]++; + s1.set_code_len(newCode[i],len); + symbols[newCode[i]] = s1; + } + // renumber the codes in byteCodes[] + for(u32 i=0; i<256; i++) + if ((byteCodes[i] & FSST_CODE_MASK) >= FSST_CODE_BASE) + byteCodes[i] = newCode[(u8) byteCodes[i]] + (1 << FSST_LEN_BITS); + else + byteCodes[i] = 511 + (1 << FSST_LEN_BITS); + + // renumber the codes in shortCodes[] + for(u32 i=0; i<65536; i++) + if ((shortCodes[i] & FSST_CODE_MASK) >= FSST_CODE_BASE) + shortCodes[i] = newCode[(u8) shortCodes[i]] + (shortCodes[i] & (15 << FSST_LEN_BITS)); + else + shortCodes[i] = byteCodes[i&0xFF]; + + // replace the symbols in the hash table + for(u32 i=0; i>8; + } + void count1Inc(u32 pos1) { + if (!count1Low[pos1]++) // increment high early (when low==0, not when low==255). This means (high > 0) <=> (cnt > 0) + count1High[pos1]++; //(0,0)->(1,1)->..->(255,1)->(0,1)->(1,2)->(2,2)->(3,2)..(255,2)->(0,2)->(1,3)->(2,3)... + } + void count2Inc(u32 pos1, u32 pos2) { + if (!count2Low[pos1][pos2]++) // increment high early (when low==0, not when low==255). This means (high > 0) <=> (cnt > 0) + // inc 4-bits high counter with 1<<0 (1) or 1<<4 (16) -- depending on whether pos2 is even or odd, repectively + count2High[pos1][(pos2)>>1] += 1 << (((pos2)&1)<<2); // we take our chances with overflow.. (4K maxval, on a 8K sample) + } + u32 count1GetNext(u32 &pos1) { // note: we will advance pos1 to the next nonzero counter in register range + // read 16-bits single symbol counter, split into two 8-bits numbers (count1Low, count1High), while skipping over zeros + u64 high = fsst_unaligned_load(&count1High[pos1]); // note: this reads 8 subsequent counters [pos1..pos1+7] + + u32 zero = high?(__builtin_ctzl(high)>>3):7UL; // number of zero bytes + high = (high >> (zero << 3)) & 255; // advance to nonzero counter + if (((pos1 += zero) >= FSST_CODE_MAX) || !high) // SKIP! advance pos2 + return 0; // all zero + + u32 low = count1Low[pos1]; + if (low) high--; // high is incremented early and low late, so decrement high (unless low==0) + return (u32) ((high << 8) + low); + } + u32 count2GetNext(u32 pos1, u32 &pos2) { // note: we will advance pos2 to the next nonzero counter in register range + // read 12-bits pairwise symbol counter, split into low 8-bits and high 4-bits number while skipping over zeros + u64 high = fsst_unaligned_load(&count2High[pos1][pos2>>1]); // note: this reads 16 subsequent counters [pos2..pos2+15] + high >>= ((pos2&1) << 2); // odd pos2: ignore the lowest 4 bits & we see only 15 counters + + u32 zero = high?(__builtin_ctzl(high)>>2):(15UL-(pos2&1UL)); // number of zero 4-bits counters + high = (high >> (zero << 2)) & 15; // advance to nonzero counter + if (((pos2 += zero) >= FSST_CODE_MAX) || !high) // SKIP! advance pos2 + return 0UL; // all zero + + u32 low = count2Low[pos1][pos2]; + if (low) high--; // high is incremented early and low late, so decrement high (unless low==0) + return (u32) ((high << 8) + low); + } + void backup1(u8 *buf) { + memcpy(buf, count1High, FSST_CODE_MAX); + memcpy(buf+FSST_CODE_MAX, count1Low, FSST_CODE_MAX); + } + void restore1(u8 *buf) { + memcpy(count1High, buf, FSST_CODE_MAX); + memcpy(count1Low, buf+FSST_CODE_MAX, FSST_CODE_MAX); + } +}; +#endif + + +#define FSST_BUFSZ (3<<19) // 768KB + +// an encoder is a symbolmap plus some bufferspace, needed during map construction as well as compression +struct Encoder { + shared_ptr symbolTable; // symbols, plus metadata and data structures for quick compression (shortCode,hashTab, etc) + union { + Counters counters; // for counting symbol occurences during map construction + u8 simdbuf[FSST_BUFSZ]; // for compression: SIMD string staging area 768KB = 256KB in + 512KB out (worst case for 256KB in) + }; +}; + +// job control integer representable in one 64bits SIMD lane: cur/end=input, out=output, pos=which string (2^9=512 per call) +struct SIMDjob { + u64 out:19,pos:9,end:18,cur:18; // cur/end is input offsets (2^18=256KB), out is output offset (2^19=512KB) +}; + +extern bool +fsst_hasAVX512(); // runtime check for avx512 capability + +extern size_t +fsst_compressAVX512( + SymbolTable &symbolTable, + u8* codeBase, // IN: base address for codes, i.e. compression output (points to simdbuf+256KB) + u8* symbolBase, // IN: base address for string bytes, i.e. compression input (points to simdbuf) + SIMDjob* input, // IN: input array (size n) with job information: what to encode, where to store it. + SIMDjob* output, // OUT: output array (size n) with job information: how much got encoded, end output pointer. + size_t n, // IN: size of arrays input and output (should be max 512) + size_t unroll); // IN: degree of SIMD unrolling + +// C++ fsst-compress function with some more control of how the compression happens (algorithm flavor, simd unroll degree) +size_t compressImpl(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], bool noSuffixOpt, bool avoidBranch, int simd); +size_t compressAuto(Encoder *encoder, size_t n, size_t lenIn[], u8 *strIn[], size_t size, u8 * output, size_t *lenOut, u8 *strOut[], int simd); +} // namespace libfsst From 603a076d567a7711a4085f80f626fd9104e917c9 Mon Sep 17 00:00:00 2001 From: arnavb Date: Tue, 25 Nov 2025 08:52:21 +0000 Subject: [PATCH 14/24] update --- NOTICE.txt | 4 ++++ cpp/src/parquet/CMakeLists.txt | 6 ++++-- cpp/src/parquet/decoder.cc | 2 +- cpp/src/parquet/encoder.cc | 21 +++++++++++++++++---- cpp/src/parquet/encoding_test.cc | 4 ++-- cpp/thirdparty/fsst/LICENSE | 21 +++++++++++++++++++++ dev/release/rat_exclude_files.txt | 1 + 7 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 cpp/thirdparty/fsst/LICENSE diff --git a/NOTICE.txt b/NOTICE.txt index 9b98364d2ab6..a27486d2a5d9 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21,6 +21,10 @@ This product includes software from the mman-win32 project * Copyright https://code.google.com/p/mman-win32/ * Licensed under the MIT License; +This product includes software from the Fast Static Symbol Table (FSST) project (MIT) + * Copyright (c) 2018-2020 CWI, TU Munich, FSU Jena + * https://github.com/cwida/fsst + This product includes software from the LevelDB project * Copyright (c) 2011 The LevelDB Authors. All rights reserved. * Use of this source code is governed by a BSD-style license that can be diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index e2b1db096339..843b90c5fd44 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -209,8 +209,10 @@ if(DEFINED ARROW_FSST_SOURCES) set_property(SOURCE ${ARROW_FSST_SOURCES} APPEND PROPERTY COMPILE_OPTIONS "$<$,$>:-Wno-error=shorten-64-to-32;-Wno-shorten-64-to-32>" "$<$,$,$>:-Wno-error=missing-declarations;-Wno-missing-declarations>" - "$<$:/wd4244>" - "$<$,$>>:-include${CMAKE_CURRENT_SOURCE_DIR}/fsst_compat.h>") + "$<$:/wd4244>") + set_property(SOURCE ${ARROW_FSST_SOURCES} APPEND PROPERTY COMPILE_OPTIONS + "$<$,$>>:-include>" + "$<$,$>>:${CMAKE_CURRENT_SOURCE_DIR}/fsst_compat.h>") endif() if(ARROW_HAVE_RUNTIME_AVX2) diff --git a/cpp/src/parquet/decoder.cc b/cpp/src/parquet/decoder.cc index 42838fd059f1..db4058b68381 100644 --- a/cpp/src/parquet/decoder.cc +++ b/cpp/src/parquet/decoder.cc @@ -53,7 +53,7 @@ #include "parquet/exception.h" #include "parquet/platform.h" #include "parquet/schema.h" -#include "fsst.h" +#include "fsst.h" // NOLINT(build/include_subdir) #include "parquet/types.h" #ifdef _MSC_VER diff --git a/cpp/src/parquet/encoder.cc b/cpp/src/parquet/encoder.cc index 00daa366f6ab..852fdece61a1 100644 --- a/cpp/src/parquet/encoder.cc +++ b/cpp/src/parquet/encoder.cc @@ -50,7 +50,7 @@ #include "parquet/exception.h" #include "parquet/platform.h" #include "parquet/schema.h" -#include "fsst.h" +#include "fsst.h" // NOLINT(build/include_subdir) #include "parquet/types.h" #ifdef _MSC_VER @@ -1778,9 +1778,11 @@ class FsstEncoder : public EncoderImpl, virtual public TypedEncoder(sizeof(fsst_decoder_t)); const int64_t length_prefix_bytes = - static_cast(unencoded_values_.size()) * static_cast(sizeof(uint32_t)); + static_cast(unencoded_values_.size()) * + static_cast(sizeof(uint32_t)); const int64_t estimated_buffer_size = - decoder_bytes + total_input_size * kFsstCompressionExpansion + length_prefix_bytes; + decoder_bytes + total_input_size * kFsstCompressionExpansion + + length_prefix_bytes; PARQUET_ASSIGN_OR_THROW(auto output_buffer, AllocateResizableBuffer(estimated_buffer_size, pool_)); @@ -1819,6 +1821,10 @@ class FsstEncoder : public EncoderImpl, virtual public TypedEncoderResize(total_output_size)); UpdateCompressionStats(total_input_size, total_output_size); + if (encoder_ != nullptr) { + fsst_destroy(encoder_); + encoder_ = nullptr; + } unencoded_values_.clear(); pending_unencoded_bytes_ = 0; unencoded_byte_array_data_bytes_ = 0; @@ -1860,7 +1866,9 @@ class FsstEncoder : public EncoderImpl, virtual public TypedEncoder(buffer->mutable_data()); int num_valid_values = ::arrow::util::internal::SpacedCompress( src, num_values, valid_bits, valid_bits_offset, buffer_ptr); @@ -1872,6 +1880,11 @@ class FsstEncoder : public EncoderImpl, virtual public TypedEncoder Date: Tue, 25 Nov 2025 09:27:27 +0000 Subject: [PATCH 15/24] lint --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 16 ++++++++++---- cpp/src/parquet/CMakeLists.txt | 23 ++++++++++++++------- cpp/src/parquet/decoder.cc | 2 +- cpp/src/parquet/fsst_compat.h | 6 +++--- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index c0c84511c1d5..978c14abb9fb 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -73,7 +73,9 @@ set(ARROW_THIRDPARTY_DEPENDENCIES ZLIB zstd) -set(fsst_SOURCE "BUNDLED" CACHE STRING "Source of fsst dependency") +set(fsst_SOURCE + "BUNDLED" + CACHE STRING "Source of fsst dependency") # For backward compatibility. We use "BOOST_SOURCE" if "Boost_SOURCE" # isn't specified and "BOOST_SOURCE" is specified. @@ -2613,16 +2615,22 @@ endif() function(build_fsst) message(STATUS "Configuring vendored FSST sources") - set(ARROW_FSST_INCLUDE_DIR "${ARROW_SOURCE_DIR}/thirdparty/fsst" PARENT_SCOPE) + set(ARROW_FSST_INCLUDE_DIR + "${ARROW_SOURCE_DIR}/thirdparty/fsst" + PARENT_SCOPE) set(ARROW_FSST_SOURCES "${ARROW_SOURCE_DIR}/thirdparty/fsst/libfsst.cpp;${ARROW_SOURCE_DIR}/thirdparty/fsst/fsst_avx512.cpp" PARENT_SCOPE) - set(FSST_VENDORED TRUE PARENT_SCOPE) + set(FSST_VENDORED + TRUE + PARENT_SCOPE) endfunction() if(ARROW_WITH_FSST) if(NOT fsst_SOURCE STREQUAL "BUNDLED") - message(FATAL_ERROR "FSST must currently be built from source. Set fsst_SOURCE=BUNDLED.") + message( + FATAL_ERROR + "FSST must currently be built from source. Set fsst_SOURCE=BUNDLED.") endif() resolve_dependency(fsst IS_RUNTIME_DEPENDENCY FALSE) endif() diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 843b90c5fd44..c9ade80a7f57 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -206,13 +206,19 @@ if(DEFINED ARROW_FSST_INCLUDE_DIR) list(APPEND PARQUET_TEST_EXTRA_INCLUDES ${ARROW_FSST_INCLUDE_DIR}) endif() if(DEFINED ARROW_FSST_SOURCES) - set_property(SOURCE ${ARROW_FSST_SOURCES} APPEND PROPERTY COMPILE_OPTIONS - "$<$,$>:-Wno-error=shorten-64-to-32;-Wno-shorten-64-to-32>" - "$<$,$,$>:-Wno-error=missing-declarations;-Wno-missing-declarations>" - "$<$:/wd4244>") - set_property(SOURCE ${ARROW_FSST_SOURCES} APPEND PROPERTY COMPILE_OPTIONS - "$<$,$>>:-include>" - "$<$,$>>:${CMAKE_CURRENT_SOURCE_DIR}/fsst_compat.h>") + set_property( + SOURCE ${ARROW_FSST_SOURCES} + APPEND + PROPERTY COMPILE_OPTIONS + "$<$,$>:-Wno-error=shorten-64-to-32;-Wno-shorten-64-to-32>" + "$<$,$,$>:-Wno-error=missing-declarations;-Wno-missing-declarations>" + "$<$:/wd4244>") + set_property( + SOURCE ${ARROW_FSST_SOURCES} + APPEND + PROPERTY COMPILE_OPTIONS + "$<$,$>>:-include>" + "$<$,$>>:${CMAKE_CURRENT_SOURCE_DIR}/fsst_compat.h>") endif() if(ARROW_HAVE_RUNTIME_AVX2) @@ -333,7 +339,8 @@ add_arrow_lib(parquet if(PARQUET_PRIVATE_INCLUDE_DIRS) foreach(_parquet_target parquet_objlib parquet_shared parquet_static) if(TARGET ${_parquet_target}) - target_include_directories(${_parquet_target} PRIVATE ${PARQUET_PRIVATE_INCLUDE_DIRS}) + target_include_directories( + ${_parquet_target} PRIVATE ${PARQUET_PRIVATE_INCLUDE_DIRS}) endif() endforeach() endif() diff --git a/cpp/src/parquet/decoder.cc b/cpp/src/parquet/decoder.cc index db4058b68381..ae82f3a78a65 100644 --- a/cpp/src/parquet/decoder.cc +++ b/cpp/src/parquet/decoder.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -29,7 +30,6 @@ #include #include #include -#include #include "arrow/array.h" #include "arrow/array/builder_binary.h" diff --git a/cpp/src/parquet/fsst_compat.h b/cpp/src/parquet/fsst_compat.h index 2da27d273011..1c3ef3fae0b5 100644 --- a/cpp/src/parquet/fsst_compat.h +++ b/cpp/src/parquet/fsst_compat.h @@ -21,13 +21,13 @@ // can be compiled with the compilers Arrow supports. #if defined(_WIN32) && !defined(_MSC_VER) -#include +# include // MinGW does not provide __cpuidex, but FSST only needs the CPUID // leaf/sub-leaf variant that __cpuid_count implements. static inline void arrow_fsst_cpuidex(int info[4], int function_id, int subfunction_id) { __cpuid_count(function_id, subfunction_id, info[0], info[1], info[2], info[3]); } -#define __cpuidex(info, function_id, subfunction_id) \ - arrow_fsst_cpuidex(info, function_id, subfunction_id) +# define __cpuidex(info, function_id, subfunction_id) \ + arrow_fsst_cpuidex(info, function_id, subfunction_id) #endif From d4176ae0885ca22d5ce99a1388b7f6bbf82de590 Mon Sep 17 00:00:00 2001 From: arnavb Date: Tue, 25 Nov 2025 09:53:07 +0000 Subject: [PATCH 16/24] lint --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 5 +- cpp/src/parquet/CMakeLists.txt | 29 +++++---- cpp/src/parquet/decoder.cc | 69 ++++++++++----------- cpp/src/parquet/encoder.cc | 26 ++++---- cpp/src/parquet/encoding_test.cc | 22 +++---- 5 files changed, 70 insertions(+), 81 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 978c14abb9fb..dddbf2271bb7 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2628,9 +2628,8 @@ endfunction() if(ARROW_WITH_FSST) if(NOT fsst_SOURCE STREQUAL "BUNDLED") - message( - FATAL_ERROR - "FSST must currently be built from source. Set fsst_SOURCE=BUNDLED.") + message(FATAL_ERROR "FSST must currently be built from source. Set fsst_SOURCE=BUNDLED." + ) endif() resolve_dependency(fsst IS_RUNTIME_DEPENDENCY FALSE) endif() diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index c9ade80a7f57..a38705cadf86 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -206,19 +206,18 @@ if(DEFINED ARROW_FSST_INCLUDE_DIR) list(APPEND PARQUET_TEST_EXTRA_INCLUDES ${ARROW_FSST_INCLUDE_DIR}) endif() if(DEFINED ARROW_FSST_SOURCES) - set_property( - SOURCE ${ARROW_FSST_SOURCES} - APPEND - PROPERTY COMPILE_OPTIONS - "$<$,$>:-Wno-error=shorten-64-to-32;-Wno-shorten-64-to-32>" - "$<$,$,$>:-Wno-error=missing-declarations;-Wno-missing-declarations>" - "$<$:/wd4244>") - set_property( - SOURCE ${ARROW_FSST_SOURCES} - APPEND - PROPERTY COMPILE_OPTIONS - "$<$,$>>:-include>" - "$<$,$>>:${CMAKE_CURRENT_SOURCE_DIR}/fsst_compat.h>") + set_property(SOURCE ${ARROW_FSST_SOURCES} + APPEND + PROPERTY COMPILE_OPTIONS + "$<$,$>:-Wno-error=shorten-64-to-32;-Wno-shorten-64-to-32>" + "$<$,$,$>:-Wno-error=missing-declarations;-Wno-missing-declarations>" + "$<$:/wd4244>") + set_property(SOURCE ${ARROW_FSST_SOURCES} + APPEND + PROPERTY COMPILE_OPTIONS + "$<$,$>>:-include>" + "$<$,$>>:${CMAKE_CURRENT_SOURCE_DIR}/fsst_compat.h>" + ) endif() if(ARROW_HAVE_RUNTIME_AVX2) @@ -339,8 +338,8 @@ add_arrow_lib(parquet if(PARQUET_PRIVATE_INCLUDE_DIRS) foreach(_parquet_target parquet_objlib parquet_shared parquet_static) if(TARGET ${_parquet_target}) - target_include_directories( - ${_parquet_target} PRIVATE ${PARQUET_PRIVATE_INCLUDE_DIRS}) + target_include_directories(${_parquet_target} + PRIVATE ${PARQUET_PRIVATE_INCLUDE_DIRS}) endif() endforeach() endif() diff --git a/cpp/src/parquet/decoder.cc b/cpp/src/parquet/decoder.cc index ae82f3a78a65..03105e4fea4d 100644 --- a/cpp/src/parquet/decoder.cc +++ b/cpp/src/parquet/decoder.cc @@ -50,10 +50,10 @@ #include "arrow/util/ubsan.h" #include "arrow/visit_data_inline.h" +#include "fsst.h" // NOLINT(build/include_subdir) #include "parquet/exception.h" #include "parquet/platform.h" #include "parquet/schema.h" -#include "fsst.h" // NOLINT(build/include_subdir) #include "parquet/types.h" #ifdef _MSC_VER @@ -2378,8 +2378,7 @@ class FsstDecoder : public DecoderImpl, virtual public TypedDecoder::Accumulator* builder) override { int values_decoded = 0; - PARQUET_THROW_NOT_OK( - DecodeArrowDense(num_values, null_count, valid_bits, valid_bits_offset, builder, - &values_decoded)); + PARQUET_THROW_NOT_OK(DecodeArrowDense(num_values, null_count, valid_bits, + valid_bits_offset, builder, &values_decoded)); return values_decoded; } - int DecodeArrow(int num_values, int null_count, const uint8_t* valid_bits, - int64_t valid_bits_offset, - typename EncodingTraits::DictAccumulator* builder) override { + int DecodeArrow( + int num_values, int null_count, const uint8_t* valid_bits, + int64_t valid_bits_offset, + typename EncodingTraits::DictAccumulator* builder) override { int values_decoded = 0; - PARQUET_THROW_NOT_OK( - DecodeArrowDict(num_values, null_count, valid_bits, valid_bits_offset, builder, - &values_decoded)); + PARQUET_THROW_NOT_OK(DecodeArrowDict(num_values, null_count, valid_bits, + valid_bits_offset, builder, &values_decoded)); return values_decoded; } @@ -2482,32 +2480,31 @@ class FsstDecoder : public DecoderImpl, virtual public TypedDecoderReserve(num_values)); int value_index = 0; - RETURN_NOT_OK(VisitBitRuns( - valid_bits, valid_bits_offset, num_values, - [&](int64_t position, int64_t run_length, bool is_valid) { - if (is_valid) { - for (int64_t i = 0; i < run_length; ++i) { - const auto& value = temp_values_[value_index++]; - RETURN_NOT_OK(builder->Append(value.ptr, static_cast(value.len))); - } - } else { - RETURN_NOT_OK(builder->AppendNulls(run_length)); - } - return Status::OK(); - })); + RETURN_NOT_OK(VisitBitRuns(valid_bits, valid_bits_offset, num_values, + [&](int64_t position, int64_t run_length, bool is_valid) { + if (is_valid) { + for (int64_t i = 0; i < run_length; ++i) { + const auto& value = temp_values_[value_index++]; + RETURN_NOT_OK(builder->Append( + value.ptr, static_cast(value.len))); + } + } else { + RETURN_NOT_OK(builder->AppendNulls(run_length)); + } + return Status::OK(); + })); *out_values_decoded = decoded; return Status::OK(); } uint8_t* EnsureDecodeBuffer(int64_t capacity) { - const int64_t min_capacity = - std::max(capacity, kInitialDecodeBufferSize); + const int64_t min_capacity = std::max(capacity, kInitialDecodeBufferSize); const int64_t target = ::arrow::bit_util::NextPower2(min_capacity); if (!decode_buffer_) { - PARQUET_ASSIGN_OR_THROW( - decode_buffer_, ::arrow::AllocateResizableBuffer(target, pool_)); + PARQUET_ASSIGN_OR_THROW(decode_buffer_, + ::arrow::AllocateResizableBuffer(target, pool_)); } else if (decode_buffer_->size() < target) { PARQUET_THROW_NOT_OK(decode_buffer_->Resize(target, false)); } @@ -2516,26 +2513,24 @@ class FsstDecoder : public DecoderImpl, virtual public TypedDecodermutable_data() + decode_buffer_size_; const size_t available = static_cast(decode_buffer_->size() - decode_buffer_size_); - const size_t decompressed = - fsst_decompress(&decoder_, compressed_len, compressed_ptr, available, - destination); + const size_t decompressed = fsst_decompress(&decoder_, compressed_len, + compressed_ptr, available, destination); if (decompressed > 0 || compressed_len == 0) { *value_ptr = destination; return decompressed; } - int64_t new_capacity = std::max( - decode_buffer_->size() * 2, - decode_buffer_size_ + OutputUpperBound(compressed_len)); + int64_t new_capacity = + std::max(decode_buffer_->size() * 2, + decode_buffer_size_ + OutputUpperBound(compressed_len)); if (new_capacity <= decode_buffer_->size()) { throw ParquetException("FSST decompression failed"); } diff --git a/cpp/src/parquet/encoder.cc b/cpp/src/parquet/encoder.cc index 852fdece61a1..e0a2f1ec8269 100644 --- a/cpp/src/parquet/encoder.cc +++ b/cpp/src/parquet/encoder.cc @@ -47,10 +47,10 @@ #include "arrow/util/ubsan.h" #include "arrow/visit_data_inline.h" +#include "fsst.h" // NOLINT(build/include_subdir) #include "parquet/exception.h" #include "parquet/platform.h" #include "parquet/schema.h" -#include "fsst.h" // NOLINT(build/include_subdir) #include "parquet/types.h" #ifdef _MSC_VER @@ -1758,10 +1758,8 @@ class FsstEncoder : public EncoderImpl, virtual public TypedEncoder(total_size) * compression_ratio_hint_; - const int64_t estimated_payload = - static_cast(std::ceil(scaled)); + const double scaled = static_cast(total_size) * compression_ratio_hint_; + const int64_t estimated_payload = static_cast(std::ceil(scaled)); return static_cast(sizeof(fsst_decoder_t)) + std::max(0, estimated_payload); } @@ -1777,12 +1775,11 @@ class FsstEncoder : public EncoderImpl, virtual public TypedEncoder(sizeof(fsst_decoder_t)); - const int64_t length_prefix_bytes = - static_cast(unencoded_values_.size()) * - static_cast(sizeof(uint32_t)); - const int64_t estimated_buffer_size = - decoder_bytes + total_input_size * kFsstCompressionExpansion + - length_prefix_bytes; + const int64_t length_prefix_bytes = static_cast(unencoded_values_.size()) * + static_cast(sizeof(uint32_t)); + const int64_t estimated_buffer_size = decoder_bytes + + total_input_size * kFsstCompressionExpansion + + length_prefix_bytes; PARQUET_ASSIGN_OR_THROW(auto output_buffer, AllocateResizableBuffer(estimated_buffer_size, pool_)); @@ -1867,8 +1864,7 @@ class FsstEncoder : public EncoderImpl, virtual public TypedEncoder(buffer->mutable_data()); int num_valid_values = ::arrow::util::internal::SpacedCompress( src, num_values, valid_bits, valid_bits_offset, buffer_ptr); @@ -1897,8 +1893,8 @@ class FsstEncoder : public EncoderImpl, virtual public TypedEncoder(Encoding::FSST); ASSERT_NO_THROW(encoder->Put(*values)); @@ -2711,13 +2711,13 @@ TEST(TestFsstEncoding, MultiPageRoundTrip) { ->build(); std::unique_ptr writer; - ASSERT_NO_THROW(writer = - ParquetFileWriter::Open(output_stream, parquet_schema, writer_props)); + ASSERT_NO_THROW( + writer = ParquetFileWriter::Open(output_stream, parquet_schema, writer_props)); ASSERT_NE(nullptr, writer); auto* row_group_writer = writer->AppendRowGroup(); ASSERT_NE(nullptr, row_group_writer); - auto* column_writer = static_cast*>( - row_group_writer->NextColumn()); + auto* column_writer = + static_cast*>(row_group_writer->NextColumn()); ASSERT_NE(nullptr, column_writer); auto write_page = [&](const std::vector& source) { @@ -2760,16 +2760,16 @@ TEST(TestFsstEncoding, MultiPageRoundTrip) { ASSERT_NO_THROW(reader = make_reader()); ASSERT_NE(nullptr, reader); auto row_group_reader = reader->RowGroup(0); - auto column_reader = - std::static_pointer_cast>(row_group_reader->Column(0)); + auto column_reader = std::static_pointer_cast>( + row_group_reader->Column(0)); std::vector decoded(kTotalValues); int64_t values_read = 0; while (values_read < kTotalValues) { int64_t batch_length = std::min(1024, kTotalValues - values_read); int64_t batch_read = 0; - column_reader->ReadBatch(batch_length, nullptr, nullptr, - decoded.data() + values_read, &batch_read); + column_reader->ReadBatch(batch_length, nullptr, nullptr, decoded.data() + values_read, + &batch_read); ASSERT_GT(batch_read, 0); values_read += batch_read; } From aef5d3cd06f1be40f5aa1390f4d9b5abecfe594d Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 3 Aug 2026 06:01:55 +0000 Subject: [PATCH 17/24] Add an OnPair string codec A C++ implementation of the OnPair short-string codec (arXiv:2508.02280): a trained dictionary of up to 16-byte tokens, greedy longest-prefix tokenization, and a branch-free gather-copy decode that keeps per-row random access. The dictionary budget is configurable from 9 to 16 bits; codes are bit-packed at the dictionary's true width. --- cpp/src/parquet/onpair/onpair.cc | 889 ++++++++++++++++++ cpp/src/parquet/onpair/onpair.h | 185 ++++ .../parquet/onpair/onpair_test_standalone.cc | 132 +++ 3 files changed, 1206 insertions(+) create mode 100644 cpp/src/parquet/onpair/onpair.cc create mode 100644 cpp/src/parquet/onpair/onpair.h create mode 100644 cpp/src/parquet/onpair/onpair_test_standalone.cc diff --git a/cpp/src/parquet/onpair/onpair.cc b/cpp/src/parquet/onpair/onpair.cc new file mode 100644 index 000000000000..2cd99c100d0c --- /dev/null +++ b/cpp/src/parquet/onpair/onpair.cc @@ -0,0 +1,889 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/onpair/onpair.h" + +#include +#include +#include +#include + +namespace parquet::onpair { +namespace { + +constexpr size_t kBucketPrefixLen = 8; +constexpr size_t kPromoteThreshold = 128; + +// Little-endian packing helpers + +/// Pack the low min(len, data_len, 8) bytes of `data` into a little-endian u64; +/// higher bytes read as zero. +inline uint64_t LoadLeU64(const uint8_t* data, size_t data_len, size_t len) { + size_t n = (len >= kBucketPrefixLen && data_len >= kBucketPrefixLen) + ? kBucketPrefixLen + : std::min(len, data_len); + uint64_t v = 0; + std::memcpy(&v, data, n); // little-endian host + return v; +} + +/// Mask of the low len*8 bits in a u64. +inline uint64_t MaskU64(size_t len) { + return len >= 8 ? ~uint64_t{0} : ((uint64_t{1} << (len * 8)) - 1); +} + +/// Count of matching low bytes between two packed suffixes. +inline size_t MatchingLowBytes(uint64_t x) { + return x == 0 ? 8 : (static_cast(__builtin_ctzll(x)) >> 3); +} + +// Flat u64 -> u32 hash table +// +// The tokenizer probes its lookup tables several times per token, so probe cost +// dominates encode time. std::unordered_map is the wrong shape for that: the +// bucket array load and the node load are dependent, so every miss costs two +// serialized cache misses, and at tens of thousands of tokens neither fits in +// cache. Open addressing with the key and value in one 16-byte slot makes the +// common case a single load. Insert order still decides which of two equal keys +// wins, so swapping this in cannot change the tokenization. + +constexpr uint32_t kFlatEmpty = ~uint32_t{0}; + +inline uint64_t MixU64(uint64_t x) { + x ^= x >> 33; + x *= 0xff51afd7ed558ccdULL; + x ^= x >> 33; + x *= 0xc4ceb9fe1a85ec53ULL; + x ^= x >> 33; + return x; +} + +class FlatU64Map { + public: + FlatU64Map() : slots_(kMinSlots), mask_(kMinSlots - 1) {} + + bool empty() const { return size_ == 0; } + + /// Value for `key`, or kFlatEmpty when absent. + uint32_t Find(uint64_t key) const { + size_t i = MixU64(key) & mask_; + for (;;) { + const Slot& s = slots_[i]; + if (s.val == kFlatEmpty) return kFlatEmpty; + if (s.key == key) return s.val; + i = (i + 1) & mask_; + } + } + + /// Insert `key`, or overwrite the value already stored under it. + void Put(uint64_t key, uint32_t val) { + size_t i = MixU64(key) & mask_; + for (;;) { + Slot& s = slots_[i]; + if (s.val == kFlatEmpty) { + s.key = key; + s.val = val; + ++size_; + // Linear probing degrades sharply past half full; grow well before then. + if (size_ * 2 > slots_.size()) Grow(); + return; + } + if (s.key == key) { + s.val = val; + return; + } + i = (i + 1) & mask_; + } + } + + private: + struct Slot { + uint64_t key = 0; + uint32_t val = kFlatEmpty; + }; + static constexpr size_t kMinSlots = 64; + + void Grow() { + std::vector old(slots_.size() * 2); + old.swap(slots_); + mask_ = slots_.size() - 1; + for (const Slot& s : old) { + if (s.val == kFlatEmpty) continue; + size_t i = MixU64(s.key) & mask_; + while (slots_[i].val != kFlatEmpty) i = (i + 1) & mask_; + slots_[i] = s; + } + } + + std::vector slots_; + size_t mask_; + size_t size_ = 0; +}; + +// Flat pair-frequency counter for the training loop +// +// The trainer touches this once per token boundary, so it sits on the same hot +// path as the matcher and wants the same treatment. Deleting a promoted pair is a +// reset to zero rather than a real erase: a caller cannot tell an absent key from +// a zero count, so the two are equivalent, and it keeps linear probing free of +// tombstones (promotions are also rare - at most one per dictionary entry). + +inline uint32_t MixU32(uint32_t x) { + x ^= x >> 16; + x *= 0x7feb352dU; + x ^= x >> 15; + x *= 0x846ca68bU; + x ^= x >> 16; + return x; +} + +class FlatFreqMap { + public: + FlatFreqMap() : slots_(kMinSlots), mask_(kMinSlots - 1) {} + + /// Saturating increment of `key`'s count (absent == 0), returning the new value. + uint8_t Bump(uint32_t key) { + size_t i = MixU32(key) & mask_; + for (;;) { + Slot& s = slots_[i]; + if (s.count == kEmptyCount) { + s.key = key; + s.count = 1; + ++size_; + if (size_ * 2 > slots_.size()) Grow(); + return 1; + } + if (s.key == key) { + if (s.count < 255) ++s.count; + return static_cast(s.count); + } + i = (i + 1) & mask_; + } + } + + /// Forget `key`'s count. Precondition: Bump(key) was called at least once. + void Reset(uint32_t key) { + size_t i = MixU32(key) & mask_; + for (;;) { + Slot& s = slots_[i]; + if (s.count == kEmptyCount) return; + if (s.key == key) { + s.count = 0; + return; + } + i = (i + 1) & mask_; + } + } + + private: + struct Slot { + uint32_t key = 0; + uint16_t count = kEmptyCount; + }; + static constexpr uint16_t kEmptyCount = 0xFFFF; + static constexpr size_t kMinSlots = 1024; + + void Grow() { + std::vector old(slots_.size() * 2); + old.swap(slots_); + mask_ = slots_.size() - 1; + for (const Slot& s : old) { + if (s.count == kEmptyCount) continue; + size_t i = MixU32(s.key) & mask_; + while (slots_[i].count != kEmptyCount) i = (i + 1) & mask_; + slots_[i] = s; + } + } + + std::vector slots_; + size_t mask_; + size_t size_ = 0; +}; + +// Longest-prefix matcher +// Two-tier index per the paper (sec 3.4.1): a hash map for tokens <=8 bytes, and +// 8-byte-prefix buckets (suffixes sorted descending) for 9..16-byte tokens. +// DEVIATION FROM PAPER (D3): the paper's OnPair16 caps each long bucket at 128 +// suffixes (sec 3.4.4, dropping extras); this port instead promotes an over-full +// bucket to a trie (PROMOTE_THRESHOLD), keeping all suffixes. Also, the paper's +// static parsing phase (sec 3.4.3) finalizes long-pattern lookup with a minimal +// perfect hash; this port keeps an ordinary hash table (the paper notes the +// perfect-hash path is Rust-only). Encode-time behavior only. + +// Prefix filter +// +// One byte per possible two-byte prefix of the data: bit (len-1) is set when some +// token of exactly `len` bytes (2..kBucketPrefixLen) starts with those two bytes, +// and bit 0 when some token longer than kBucketPrefixLen does. Length 1 is left +// out, since a single-byte match always exists and is probed anyway. Packing the +// eight live bits into a byte rather than a u16 halves the table to 64 KB. +// +// Folding the prefix into fewer slots would stay correct - the filter only ever +// SKIPS work, so a collision costs a wasted probe and can never change the answer +// - but measurably loses: at 16 KB and below the index arithmetic costs more than +// the smaller footprint saves, because the live prefix set is already small. + +constexpr size_t kPrefixSlots = size_t{1} << 16; +constexpr uint8_t kMaskLongBit = 1; + +struct LongEntry { + uint64_t suffix; + uint8_t slen; + Token token; +}; + +struct TrieNode { + int token = -1; // -1 == none + std::vector> children; +}; + +struct Bucket { + std::vector entries; + int32_t trie_root = -1; // >=0 once promoted +}; + +class LongestPrefixMatcher { + public: + /// Empty matcher pre-loaded with the 256 single-byte tokens (ids 0..255). + static LongestPrefixMatcher New() { + LongestPrefixMatcher m; + for (uint16_t i = 0; i <= 255; ++i) { + m.short_by_len_[1].Put(static_cast(static_cast(i)), i); + } + m.next_id_ = 256; + return m; + } + + /// Build from a complete dictionary: token at index i receives id i. + static LongestPrefixMatcher FromDictionary(const CompactDictionary& dict) { + LongestPrefixMatcher m; + size_t n = dict.num_tokens(); + for (size_t i = 0; i < n; ++i) { + m.InsertInternal(dict.token_ptr(static_cast(i)), dict.token_len(static_cast(i)), + static_cast(i)); + } + m.next_id_ = static_cast(n); + return m; + } + + /// Insert `data` (len bytes) and assign it the next available token id. + Token Insert(const uint8_t* data, size_t len) { + Token id = static_cast(next_id_++); + InsertInternal(data, len, id); + return id; + } + + size_t size() const { return next_id_; } + + /// Longest token whose bytes are a prefix of `data`, with its length. + std::pair FindLongestMatch(const uint8_t* data, size_t data_len) const { + size_t max_len = std::min(data_len, kMaxTokenSize); + uint64_t low64 = LoadLeU64(data, data_len, std::min(max_len, kBucketPrefixLen)); + + // Every token of 2 bytes or more shares its first two bytes with the data, so + // one array read rules out the lengths at which no token can possibly match. + uint32_t present = max_len >= 2 ? prefix_mask_[low64 & 0xFFFF] : 0; + + if (max_len > kBucketPrefixLen && (present & kMaskLongBit) != 0) { + uint32_t bucket = long_map_.Find(low64); + if (bucket != kFlatEmpty) { + const uint8_t* suf = data + kBucketPrefixLen; + size_t suf_len = max_len - kBucketPrefixLen; + const Bucket& b = buckets_[bucket]; + std::pair hit{0, 0}; + bool found; + if (b.trie_root < 0) { + found = SearchLinear(b.entries, LoadLeU64(suf, suf_len, suf_len), suf_len, &hit); + } else { + found = SearchTrie(static_cast(b.trie_root), suf, suf_len, &hit); + } + if (found) { + return {hit.first, kBucketPrefixLen + hit.second}; + } + } + } + + // Descend only through the occupied lengths. Bit (len-1) holds length `len`, + // so clearing bit 0 also drops the long-token bit. + size_t short_max = std::min(max_len, kBucketPrefixLen); + uint32_t cand = present & (((uint32_t{1} << short_max) - 1) & ~uint32_t{1}); + while (cand != 0) { + size_t len = 32 - static_cast(__builtin_clz(cand)); + cand &= ~(uint32_t{1} << (len - 1)); + uint32_t tok = short_by_len_[len].Find(low64 & MaskU64(len)); + if (tok != kFlatEmpty) { + return {static_cast(tok), len}; + } + } + uint32_t one = short_by_len_[1].Find(low64 & 0xFF); + if (one != kFlatEmpty) return {static_cast(one), 1}; + // Precondition: every single-byte token is present, so the probe above hits. + return {static_cast(data[0]), 1}; + } + + private: + // short_by_len_[len] maps the low-`len`-byte packed key to a token, for len 1..8. + FlatU64Map short_by_len_[kBucketPrefixLen + 1]; + FlatU64Map long_map_; // 8-byte prefix -> index into buckets_ + std::vector buckets_; + std::vector pool_; + std::vector prefix_mask_ = std::vector(kPrefixSlots, 0); + uint32_t next_id_ = 0; + + void InsertInternal(const uint8_t* data, size_t len, Token id) { + if (len >= 2) { + uint32_t p = static_cast(data[0]) | (static_cast(data[1]) << 8); + uint32_t bit = len <= kBucketPrefixLen ? (uint32_t{1} << (len - 1)) : kMaskLongBit; + prefix_mask_[p] |= static_cast(bit); + } + if (len <= kBucketPrefixLen) { + uint64_t key = LoadLeU64(data, len, len); + short_by_len_[len].Put(key, id); + return; + } + uint64_t prefix = LoadLeU64(data, len, kBucketPrefixLen); + size_t slen = len - kBucketPrefixLen; + uint64_t suffix = LoadLeU64(data + kBucketPrefixLen, slen, slen); + uint32_t bi = long_map_.Find(prefix); + if (bi == kFlatEmpty) { + bi = static_cast(buckets_.size()); + buckets_.emplace_back(); + long_map_.Put(prefix, bi); + } + Bucket& b = buckets_[bi]; + if (b.trie_root < 0) { + // Keep descending-by-length order so the first linear match is longest. The + // bucket is already ordered, so place the new entry rather than re-sorting + // the whole thing on every insert - this runs inside the training loop. + // Two entries can only share a length if they also share a suffix, i.e. if + // the same token bytes were inserted twice, so where equal lengths land + // relative to each other is not observable. + LongEntry e{suffix, static_cast(slen), id}; + auto by_len_desc = [](const LongEntry& a, const LongEntry& c) { + return a.slen > c.slen; + }; + auto at = std::upper_bound(b.entries.begin(), b.entries.end(), e, by_len_desc); + b.entries.insert(at, e); + if (b.entries.size() > kPromoteThreshold) { + BuildTrie(&b); + } + } else { + uint8_t buf[8]; + std::memcpy(buf, &suffix, 8); + TrieInsert(static_cast(b.trie_root), buf, slen, id); + } + } + + bool SearchLinear(const std::vector& entries, uint64_t val, size_t max_slen, + std::pair* out) const { + for (const LongEntry& e : entries) { + size_t elen = e.slen; + if (elen <= max_slen && MatchingLowBytes(val ^ e.suffix) >= elen) { + *out = {e.token, elen}; + return true; + } + } + return false; + } + + bool SearchTrie(uint32_t root, const uint8_t* suf, size_t suf_len, + std::pair* out) const { + bool have = false; + uint32_t cur = root; + for (size_t pos = 0; pos < suf_len; ++pos) { + uint32_t child; + if (!TrieFindChild(cur, suf[pos], &child)) break; + cur = child; + if (pool_[cur].token >= 0) { + *out = {static_cast(pool_[cur].token), pos + 1}; + have = true; + } + } + return have; + } + + bool TrieFindChild(uint32_t node, uint8_t byte, uint32_t* out) const { + for (const auto& kv : pool_[node].children) { + if (kv.first == byte) { + *out = kv.second; + return true; + } + } + return false; + } + + uint32_t TrieAlloc() { + uint32_t idx = static_cast(pool_.size()); + pool_.emplace_back(); + return idx; + } + + void TrieInsert(uint32_t root, const uint8_t* suf, size_t slen, Token token) { + uint32_t cur = root; + for (size_t i = 0; i < slen; ++i) { + uint32_t child; + if (TrieFindChild(cur, suf[i], &child)) { + cur = child; + } else { + uint32_t new_idx = TrieAlloc(); + pool_[cur].children.emplace_back(suf[i], new_idx); + cur = new_idx; + } + } + pool_[cur].token = static_cast(token); + } + + void BuildTrie(Bucket* b) { + uint32_t root = TrieAlloc(); + for (const LongEntry& e : b->entries) { + uint8_t buf[8]; + std::memcpy(buf, &e.suffix, 8); + TrieInsert(root, buf, e.slen, e.token); + } + b->entries.clear(); + b->entries.shrink_to_fit(); + b->trie_root = static_cast(root); + } +}; + +// Merge-threshold controller - DEVIATION FROM PAPER (D1; see onpair.h) + +class DynamicThresholdController { + public: + DynamicThresholdController(size_t capacity, size_t total_bytes, double scan_fraction) + : capacity_(capacity), + scan_budget_(static_cast(static_cast(total_bytes) * scan_fraction)), + check_interval_(std::max(capacity / 128, 64)), + next_checkpoint_(check_interval_) {} + + uint8_t get() const { return threshold_; } + bool budget_exhausted() const { return bytes_scanned_ > scan_budget_; } + void on_bytes_scanned(size_t n) { bytes_scanned_ += n; } + + void on_entry_created() { + ++entries_created_; + if (entries_created_ >= next_checkpoint_) Rebalance(); + } + + private: + size_t capacity_; + size_t scan_budget_; + size_t check_interval_; + uint8_t threshold_ = 2; + size_t entries_created_ = 0; + size_t bytes_scanned_ = 0; + size_t entries_at_check_ = 0; + size_t bytes_at_check_ = 0; + size_t next_checkpoint_; + + void Rebalance() { + size_t delta_e = entries_created_ - entries_at_check_; + size_t delta_b = bytes_scanned_ - bytes_at_check_; + double recent_rate = + delta_b > 0 ? static_cast(delta_e) / static_cast(delta_b) : 1e9; + size_t e_rem = capacity_ > entries_created_ ? capacity_ - entries_created_ : 1; + size_t b_rem = scan_budget_ > bytes_scanned_ ? scan_budget_ - bytes_scanned_ : 1; + double target_rate = static_cast(e_rem) / static_cast(b_rem); + double ratio = target_rate > 0.0 ? recent_rate / target_rate : 1e9; + + if (ratio > 2.0 && threshold_ < 255) { + ++threshold_; + } else if (ratio < 0.5 && threshold_ > 2) { + --threshold_; + } + entries_at_check_ = entries_created_; + bytes_at_check_ = bytes_scanned_; + next_checkpoint_ = entries_created_ + check_interval_; + } +}; + +// Seeded PRNG for training-sample shuffle - DEVIATION FROM PAPER (D4) + +inline uint64_t SplitMix64(uint64_t* state) { + uint64_t z = (*state += 0x9E3779B97F4A7C15ull); + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); +} + +/// Partial Fisher-Yates: randomize the first `k` positions of `order`. +void PartialShuffle(std::vector* order, size_t k, uint64_t seed) { + size_t n = order->size(); + uint64_t state = seed; + size_t limit = std::min(k, n); + for (size_t i = 0; i < limit; ++i) { + size_t span = n - i; + size_t j = i + static_cast(SplitMix64(&state) % span); + std::swap((*order)[i], (*order)[j]); + } +} + +// Dictionary finalization + +/// Sort tokens bytewise-lexicographically, returning fresh (bytes, offsets). +void SortTokens(const std::vector& bytes, const std::vector& offsets, + std::vector* out_bytes, std::vector* out_offsets) { + size_t n = offsets.size() - 1; + auto tok_begin = [&](size_t id) { return bytes.data() + offsets[id]; }; + auto tok_len = [&](size_t id) { return offsets[id + 1] - offsets[id]; }; + + std::vector perm(n); + std::iota(perm.begin(), perm.end(), 0); + std::sort(perm.begin(), perm.end(), [&](size_t a, size_t b) { + size_t la = tok_len(a), lb = tok_len(b); + int cmp = std::memcmp(tok_begin(a), tok_begin(b), std::min(la, lb)); + if (cmp != 0) return cmp < 0; + return la < lb; + }); + + out_bytes->clear(); + out_bytes->reserve(bytes.size()); + out_offsets->clear(); + out_offsets->reserve(n + 1); + out_offsets->push_back(0); + for (size_t old : perm) { + out_bytes->insert(out_bytes->end(), tok_begin(old), tok_begin(old) + tok_len(old)); + out_offsets->push_back(static_cast(out_bytes->size())); + } +} + +/// Append zero padding so the fixed 16-byte over-read of any token is in bounds. +void PadRaw(std::vector* bytes, const std::vector& offsets) { + size_t need = static_cast(offsets.back()) + kMaxTokenSize; + if (bytes->size() < need) bytes->resize(need, 0); +} + +// Dictionary construction / training (paper sec 3.2) + +struct TrainResult { + CompactDictionary dict; + LongestPrefixMatcher lpm; +}; + +TrainResult Train(const uint8_t* data, const uint32_t* offsets, size_t n, + const Config& cfg, EncodeProfile* profile) { + using Clock = std::chrono::steady_clock; + auto t0 = Clock::now(); + size_t dict_capacity = size_t{1} << cfg.max_dict_bits; + + std::vector dict_bytes; + dict_bytes.reserve(dict_capacity * kMaxTokenSize); + std::vector dict_offsets; + dict_offsets.reserve(dict_capacity + 1); + dict_offsets.push_back(0); + for (uint16_t i = 0; i <= 255; ++i) { + dict_bytes.push_back(static_cast(i)); + dict_offsets.push_back(static_cast(dict_bytes.size())); + } + LongestPrefixMatcher lpm = LongestPrefixMatcher::New(); + + size_t total_bytes = n == 0 ? 0 : offsets[n]; + size_t capacity = dict_capacity - 256; + DynamicThresholdController ctrl(capacity, total_bytes, cfg.threshold_fraction); + uint8_t threshold = ctrl.get(); + + std::vector order(n); + std::iota(order.begin(), order.end(), 0u); + // Full Fisher-Yates shuffle of the entire training order (D4 in onpair.h). The + // dynamic byte budget still stops scanning well before the end, so only a + // sample is trained on - but drawing that sample from a *full* shuffle avoids + // skew on sequentially-ordered columns. (The Rust reference crate partial-shuffles + // only ~0.3n rows and leaves them in the slice's TAIL while the trainer reads from + // the head, so on ordered data like Customer#000... it trains mostly on + // low-numbered rows and builds a skewed dictionary. This port already shuffles + // into the head; a full shuffle matches the reference C++ std::shuffle over all + // rows and removes any doubt.) + PartialShuffle(&order, n, cfg.seed); + + FlatFreqMap freq; + + bool full_dictionary = false; + bool budget_exhausted = false; + + for (uint32_t idx : order) { + if (full_dictionary || budget_exhausted) break; + + size_t s_start = offsets[idx]; + size_t s_end = offsets[idx + 1]; + if (s_end == s_start) continue; + const uint8_t* str = data + s_start; + size_t len = s_end - s_start; + + auto [prev_id, prev_len] = lpm.FindLongestMatch(str, len); + size_t pos = prev_len; + + ctrl.on_bytes_scanned(prev_len); + if (ctrl.budget_exhausted()) { + budget_exhausted = true; + break; + } + + while (pos < len) { + auto [curr_id, curr_len] = lpm.FindLongestMatch(str + pos, len - pos); + + ctrl.on_bytes_scanned(curr_len); + if (ctrl.budget_exhausted()) { + budget_exhausted = true; + break; + } + + size_t pair_len = prev_len + curr_len; + if (pair_len <= kMaxTokenSize) { + uint32_t key = (static_cast(prev_id) << 16) | static_cast(curr_id); + uint8_t count = freq.Bump(key); + if (count >= threshold) { + size_t pair_start = pos - prev_len; + Token new_id = lpm.Insert(str + pair_start, pair_len); + dict_bytes.insert(dict_bytes.end(), str + pair_start, str + pos + curr_len); + dict_offsets.push_back(static_cast(dict_bytes.size())); + + if (lpm.size() == dict_capacity) { + full_dictionary = true; + break; + } + ctrl.on_entry_created(); + threshold = ctrl.get(); + + freq.Reset(key); + prev_id = new_id; + prev_len = pair_len; + pos += curr_len; + continue; + } + } + prev_id = curr_id; + prev_len = curr_len; + pos += curr_len; + } + } + + std::vector sorted_bytes; + std::vector sorted_offsets; + auto t1 = Clock::now(); + SortTokens(dict_bytes, dict_offsets, &sorted_bytes, &sorted_offsets); + PadRaw(&sorted_bytes, sorted_offsets); + + CompactDictionary dict; + dict.bytes = std::move(sorted_bytes); + dict.offsets = std::move(sorted_offsets); + dict.RecomputeMaxTokenLen(); + LongestPrefixMatcher final_lpm = LongestPrefixMatcher::FromDictionary(dict); + if (profile != nullptr) { + profile->train_s = std::chrono::duration(t1 - t0).count(); + profile->rebuild_s = std::chrono::duration(Clock::now() - t1).count(); + } + return TrainResult{std::move(dict), std::move(final_lpm)}; +} + +// Parsing: greedy longest-prefix tokenization (paper sec 3.3) + +void EncodeStrings(const uint8_t* data, const uint32_t* offsets, size_t n, + const LongestPrefixMatcher& lpm, std::vector* codes, + std::vector* row_offsets) { + row_offsets->push_back(0); + for (size_t i = 0; i < n; ++i) { + size_t s = offsets[i]; + size_t e = offsets[i + 1]; + size_t pos = s; + while (pos < e) { + auto [tok, mlen] = lpm.FindLongestMatch(data + pos, e - pos); + codes->push_back(tok); + pos += mlen; + } + row_offsets->push_back(static_cast(codes->size())); + } +} + +} // namespace + +// Public API + +Column Compress(const uint8_t* bytes, size_t /*bytes_len*/, const uint32_t* offsets, + size_t num_rows, const Config& cfg, EncodeProfile* profile) { + TrainResult tr = Train(bytes, offsets, num_rows, cfg, profile); + Column col; + col.dict = std::move(tr.dict); + col.codes.reserve(num_rows == 0 ? 0 : offsets[num_rows]); + col.row_offsets.reserve(num_rows + 1); + auto t0 = std::chrono::steady_clock::now(); + EncodeStrings(bytes, offsets, num_rows, tr.lpm, &col.codes, &col.row_offsets); + if (profile != nullptr) { + profile->tokenize_s = + std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + } + return col; +} + +size_t DecodedLen(const Column& col) { + size_t sum = 0; + for (uint16_t c : col.codes) sum += col.dict.token_len(c); + return sum; +} + +namespace { + +// Shared body of DecompressInto, parameterised on the gather-copy width for the +// same reason DecompressPackedFixed is. See CompactDictionary::max_token_len. +template +size_t DecompressIntoFixed(const Column& col, uint8_t* out) { + const CompactDictionary& dict = col.dict; + size_t w = 0; + for (uint16_t code : col.codes) { + const uint8_t* src = dict.token_ptr(code); + size_t len = dict.token_len(code); + std::memcpy(out + w, src, kCopy); // fixed over-copy, kCopy >= every token + w += len; + } + return w; +} + +} // namespace + +size_t DecompressInto(const Column& col, uint8_t* out) { + const size_t maxlen = col.dict.max_token_len; + if (maxlen <= 4) return DecompressIntoFixed<4>(col, out); + if (maxlen <= 8) return DecompressIntoFixed<8>(col, out); + return DecompressIntoFixed(col, out); +} + +std::vector PackValues(const uint32_t* vals, size_t n, size_t bits) { + std::vector out((n * bits + 7) / 8 + 4, 0); + size_t bitpos = 0; + for (size_t i = 0; i < n; ++i) { + size_t byte = bitpos >> 3, off = bitpos & 7; + uint32_t w; + std::memcpy(&w, out.data() + byte, 4); + w |= (vals[i] << off); // vals[i] < 2^bits, bits<=25, off<=7 -> fits in u32 + std::memcpy(out.data() + byte, &w, 4); + bitpos += bits; + } + return out; +} + +namespace { + +// The gather-copy writes a fixed width per token so the copy length is a compile +// time constant, but that width only has to cover the longest token this +// dictionary actually holds -- not kMaxTokenSize. On corpora whose tokens are +// short the difference dominates decode: c_address averages 1.99 bytes per token, +// so a 16-byte copy moves 8x the bytes it needs to. +// +// Measured, this loop is store-bandwidth-bound. Across five unrelated corpora +// (over-copy factor) x (decode MiB/s) came out constant at ~10.6 GiB/s of store +// traffic, and the corpora with the highest over-copy decode slowest. Narrowing +// the width is therefore worth close to the bytes it saves. +// +// The width is chosen once per stream from the dictionary, so there is no +// per-token branch: a predicate on token length would be nearly free on corpora +// where it always goes one way and expensive on the ones that split (urls sit at +// 41% short, the worst possible mix). +template +size_t DecompressPackedFixed(const CompactDictionary& dict, const uint8_t* packed, size_t ncodes, + size_t bits, uint8_t* out) { + size_t bitpos = 0, w = 0; + const uint32_t mask = (bits >= 32) ? 0xFFFFFFFFu : ((1u << bits) - 1); + for (size_t i = 0; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, packed + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & mask; // unpack the code + bitpos += bits; + const uint8_t* src = dict.token_ptr(static_cast(code)); + size_t len = dict.token_len(static_cast(code)); + std::memcpy(out + w, src, kCopy); // fixed over-copy, kCopy >= every token + w += len; + } + return w; +} + +// As above but with the code width a compile-time constant, so the mask folds to a +// literal and `bitpos += kBits` strength-reduces. Dispatched once per stream, the +// same way the copy width is. +// +// Tried and rejected here, so it is not re-attempted: unpacking codes a block at a +// time before gathering, to break the `w += len` store-address dependency and to +// prefetch the token bytes. It lost 22% with the offsets prefetched and 41% with +// dict.bytes prefetched (worst case -65%), across all 20 corpora. The premise was +// wrong -- this loop is store-bound, not latency-bound, which is the same thing the +// copy-width measurement showed. Breaking a dependency chain buys nothing against a +// store-bandwidth limit, and the extra pass plus 64 prefetches per block only add +// traffic. +template +size_t DecompressPackedFixedBits(const CompactDictionary& dict, const uint8_t* packed, + size_t ncodes, uint8_t* out) { + constexpr uint32_t kMask = (kBits >= 32) ? 0xFFFFFFFFu : ((uint32_t{1} << kBits) - 1); + const uint8_t* offsets_raw = reinterpret_cast(dict.offsets.data()); + const uint8_t* dict_bytes = dict.bytes.data(); + size_t bitpos = 0, w = 0; + for (size_t i = 0; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, packed + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & kMask; + bitpos += kBits; + // offsets[code] and offsets[code + 1] are adjacent u32s, so one 8-byte load + // yields the token's start and end together. token_ptr/token_len would issue + // two loads for what is almost always a single cache line. + uint64_t pair; + std::memcpy(&pair, offsets_raw + size_t{code} * sizeof(uint32_t), sizeof(pair)); + const uint32_t start = static_cast(pair); + const size_t len = static_cast(pair >> 32) - start; + std::memcpy(out + w, dict_bytes + start, kCopy); + w += len; + } + return w; +} + +// Resolve `bits` to a constant for the widths a trained dictionary can produce +// (kMinDictBits..kMaxDictBits), falling back to the runtime-width loop otherwise so +// no input is rejected. +template +size_t DecompressPackedDispatchBits(const CompactDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out) { + switch (bits) { + case 9: return DecompressPackedFixedBits(dict, packed, ncodes, out); + case 10: return DecompressPackedFixedBits(dict, packed, ncodes, out); + case 11: return DecompressPackedFixedBits(dict, packed, ncodes, out); + case 12: return DecompressPackedFixedBits(dict, packed, ncodes, out); + case 13: return DecompressPackedFixedBits(dict, packed, ncodes, out); + case 14: return DecompressPackedFixedBits(dict, packed, ncodes, out); + case 15: return DecompressPackedFixedBits(dict, packed, ncodes, out); + case 16: return DecompressPackedFixedBits(dict, packed, ncodes, out); + default: return DecompressPackedFixed(dict, packed, ncodes, bits, out); + } +} + +} // namespace + +size_t DecompressPacked(const CompactDictionary& dict, const uint8_t* packed, size_t ncodes, + size_t bits, uint8_t* out) { + // Read the width, do not scan for it: an O(tokens) scan here costs 1-3% on + // dictionaries of 20-60k tokens, which is charged to decode for something a + // stored format keeps in its header. See CompactDictionary::max_token_len. + const size_t maxlen = dict.max_token_len; + // Only widths a single store can carry. A 12-byte copy moves 25% fewer bytes + // than 16 but needs two stores, and measured that loses 4-6% on every corpus it + // applied to (c_mktsegment, c_phone, p_container) -- so this is not purely a + // bandwidth effect and one wide store beats two narrow ones. Narrowing to 8 is + // worth 25-28% on the corpora that allow it. + // + // `out` needs kDecodePadding of slack either way, and dict.bytes is read-padded + // by kMaxTokenSize, so every width here is in bounds. + if (maxlen <= 4) return DecompressPackedDispatchBits<4>(dict, packed, ncodes, bits, out); + if (maxlen <= 8) return DecompressPackedDispatchBits<8>(dict, packed, ncodes, bits, out); + return DecompressPackedDispatchBits(dict, packed, ncodes, bits, out); +} + +} // namespace parquet::onpair diff --git a/cpp/src/parquet/onpair/onpair.h b/cpp/src/parquet/onpair/onpair.h new file mode 100644 index 000000000000..a0783b27caa2 --- /dev/null +++ b/cpp/src/parquet/onpair/onpair.h @@ -0,0 +1,185 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// C++ implementation of the OnPair short-string compression codec (encode + +// decode paths only), for a like-for-like comparison against FSST inside Arrow's +// benchmark harness. +// +// Algorithm: F. Gargiulo and R. Venturini, "OnPair: Short Strings Compression +// for Fast Random Access," arXiv:2508.02280, 2025. This implements the paper's +// core scheme: a dictionary of the 256 single bytes plus frequent merged pairs, +// greedy longest-prefix tokenization into u16 codes, and a table-lookup decode. +// +// NOT a production Parquet encoder - this is a benchmark artifact. +// +// Conformant with the paper: the two-tier longest-prefix index (sec 3.4.1, a hash +// map for <=8-byte tokens + 8-byte-prefix buckets with suffixes sorted +// descending), the 16-byte max token of OnPair16 (sec 3.2.2), and the fixed-16-byte +// SIMD gather-copy decode (sec 3.5, Alg. 3) - advance by the token's true length, +// relying on 16-byte source read-padding and output write-padding. +// +// DEVIATIONS FROM THE PAPER (engineering choices; they do not affect the code +// format or correctness, only the trained dictionary and encode-time behavior): +// D1. Merge threshold. The paper (sec 3.2.1) fixes it per dataset as +// max(2, floor(log2(S_MiB))). This port instead uses an adaptive +// controller paced to a byte budget (`DynamicThresholdController`), so the +// trained dictionary differs from the paper's. +// D2. Long-bucket overflow. The paper's OnPair16 caps each bucket at 128 +// suffixes (sec 3.4.4), dropping extras; this port promotes an over-full +// bucket to a trie (`PROMOTE_THRESHOLD`), keeping all suffixes. +// D3. Static perfect-hash LPM. The paper finalizes long-pattern lookup with a +// minimal perfect hash for the read-only parsing phase (sec 3.4.3); this port +// keeps std::unordered_map (the paper notes that path is Rust-only). +// D4. Training-sample selection uses a fixed-seed splitmix64 *full* +// Fisher-Yates shuffle (`PartialShuffle` over all rows). Shuffle *extent*, +// not the RNG choice, is what affects the trained dictionary: a full +// shuffle avoids skew on sequentially-ordered columns (the Rust crate +// partial-shuffles only a tail prefix, skewing patterned data like +// Customer#000…). The exact RNG is not specified by the paper, so output +// is deterministic but not bit-identical to the crate. +// +// Little-endian hosts only. + +#pragma once + +#include +#include +#include +#include + +namespace parquet::onpair { + +/// A dictionary entry id and, equivalently, a code in the code stream. +using Token = uint16_t; + +/// Maximum byte length of any dictionary token, and the fixed width the decoder +/// over-reads per token. +constexpr size_t kMaxTokenSize = 16; + +/// Trailing slack an output buffer needs beyond the decoded length: the decoder +/// over-stores a fixed 16-byte chunk for the final token. +constexpr size_t kDecodePadding = kMaxTokenSize; + +/// Training configuration. Mirrors the reference `Config`. +struct Config { + /// Dictionary-size budget: at most 2^max_dict_bits tokens. Valid range 9..=16. + uint8_t max_dict_bits = 12; + /// Dynamic-threshold byte-sampling fraction, in (0, 1]. + double threshold_fraction = 0.15; + /// Deterministic sampling seed. + uint64_t seed = 42; + + static Config Dict12() { return Config{12, 0.15, 42}; } + static Config Dict16() { return Config{16, 0.15, 42}; } +}; + +/// The token table a code stream indexes into: Arrow-binary layout (flat bytes + +/// u32 offsets). `bytes` is read-padded by kMaxTokenSize so the decoder's fixed +/// 16-byte over-read stays in bounds. +struct CompactDictionary { + std::vector bytes; // read-padded + std::vector offsets; // length num_tokens + 1 + + /// Length of the longest token present, which is what the decoder's gather-copy + /// sizes its fixed copy width from. Often well below kMaxTokenSize: TPC-H + /// c_address tops out at 5 bytes, and copying 16 there moves 8x the bytes it + /// needs to. + /// + /// This must never UNDERSTATE the true maximum -- doing so would make the + /// decoder copy less than a token's length and silently truncate. It therefore + /// defaults to the conservative kMaxTokenSize, so a dictionary that never calls + /// RecomputeMaxTokenLen still decodes correctly and merely forgoes the + /// narrowing. A stored format would carry this in its header rather than + /// recompute it, which is why the decoder reads it instead of scanning: an + /// O(tokens) scan per decode call costs 1-3% on dictionaries of 20-60k tokens. + size_t max_token_len = kMaxTokenSize; + + /// Derive max_token_len from `offsets`. Call after building or replacing them. + void RecomputeMaxTokenLen() { + size_t m = 0; + for (size_t t = 0; t + 1 < offsets.size(); ++t) { + const size_t len = offsets[t + 1] - offsets[t]; + if (len > m) m = len; + } + // An empty dictionary decodes nothing; stay conservative rather than pick a + // width from no evidence. + max_token_len = m == 0 ? kMaxTokenSize : m; + } + + size_t num_tokens() const { return offsets.empty() ? 0 : offsets.size() - 1; } + const uint8_t* token_ptr(Token id) const { return bytes.data() + offsets[id]; } + size_t token_len(Token id) const { return offsets[id + 1] - offsets[id]; } + /// Logical (unpadded) byte size of the dictionary blob. + size_t logical_bytes() const { return offsets.empty() ? 0 : offsets.back(); } +}; + +/// A compressed string column. `codes` is the row-concatenated code stream; +/// row k is codes[row_offsets[k] .. row_offsets[k+1]]. +struct Column { + CompactDictionary dict; + std::vector codes; + std::vector row_offsets; // length num_rows + 1 + + size_t num_rows() const { + return row_offsets.empty() ? 0 : row_offsets.size() - 1; + } +}; + +/// Wall-clock seconds spent in each phase of Compress. The three sum to the +/// whole call. Only for attributing encode cost; pass null in a timed run. +struct EncodeProfile { + double train_s = 0; ///< greedy pairing pass over the shuffled sample + double rebuild_s = 0; ///< sort the dictionary, rebuild the matcher over it + double tokenize_s = 0; ///< tokenize every row against the frozen dictionary +}; + +/// Train a dictionary against (bytes, offsets) and greedily tokenize every row. +/// `offsets` has length num_rows + 1; row i is bytes[offsets[i]..offsets[i+1]]. +Column Compress(const uint8_t* bytes, size_t bytes_len, const uint32_t* offsets, + size_t num_rows, const Config& cfg, EncodeProfile* profile = nullptr); + +/// Exact decoded byte length of the whole column (sum of token lengths). +size_t DecodedLen(const Column& col); + +/// Decode the whole column into `out`, returning bytes written. +/// Precondition: out capacity >= DecodedLen(col) + kDecodePadding. +size_t DecompressInto(const Column& col, uint8_t* out); + +// --- Bit-packed code stream (what a real stored format uses) ---------------- +// The in-memory Column holds u16 codes; on storage the code stream is packed at +// the true code width (ceil(log2 num_tokens)). These pack/unpack it so decode +// pays the real unpacking cost, keeping ratio and decode mutually consistent. + +/// Read `nbits` (<=25) at bit offset `bitpos`, little-endian / LSB-first. +/// `p` must have 4 readable bytes at the containing word. +inline uint32_t GetBits(const uint8_t* p, size_t bitpos, size_t nbits) { + uint32_t w; + std::memcpy(&w, p + (bitpos >> 3), 4); + return (w >> (bitpos & 7)) & (nbits >= 32 ? 0xFFFFFFFFu : ((1u << nbits) - 1)); +} + +/// Pack `n` values (each < 2^bits, bits in 1..=25) LSB-first. The result has 4 +/// trailing pad bytes (so a 4-byte window at the last value is in bounds); the +/// logical size is (n*bits+7)/8. +std::vector PackValues(const uint32_t* vals, size_t n, size_t bits); + +/// Decode a bit-packed code stream: read `bits` per code and gather-copy the +/// token. `packed` needs >=4 trailing pad bytes; `out` >= DecodedLen + padding. +size_t DecompressPacked(const CompactDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out); + +} // namespace parquet::onpair diff --git a/cpp/src/parquet/onpair/onpair_test_standalone.cc b/cpp/src/parquet/onpair/onpair_test_standalone.cc new file mode 100644 index 000000000000..59342e068cf7 --- /dev/null +++ b/cpp/src/parquet/onpair/onpair_test_standalone.cc @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Standalone roundtrip validation for the OnPair C++ port. No Arrow deps. +// Build: g++ -std=c++17 -O2 -I cpp/src onpair.cc onpair_test_standalone.cc -o t +// (run from repo root, adjust -I to reach parquet/onpair/onpair.h) + +#include +#include +#include +#include +#include + +#include "parquet/onpair/onpair.h" + +namespace op = parquet::onpair; + +namespace { + +int g_failures = 0; + +// Pack a set of rows into (bytes, offsets) and roundtrip through OnPair. +bool Roundtrip(const std::vector& rows, uint8_t bits, const char* name) { + std::vector bytes; + std::vector offsets; + offsets.push_back(0); + for (const auto& r : rows) { + bytes.insert(bytes.end(), r.begin(), r.end()); + offsets.push_back(static_cast(bytes.size())); + } + op::Config cfg; + cfg.max_dict_bits = bits; + cfg.threshold_fraction = 0.5; + cfg.seed = 42; + + op::Column col = op::Compress(bytes.data(), bytes.size(), offsets.data(), + rows.size(), cfg); + + // Whole-column decode. + size_t dlen = op::DecodedLen(col); + std::vector out(dlen + op::kDecodePadding, 0); + size_t w = op::DecompressInto(col, out.data()); + bool ok = (w == bytes.size()) && (std::memcmp(out.data(), bytes.data(), bytes.size()) == 0); + if (!ok) { + std::printf(" FAIL %-22s bits=%2u: decoded %zu vs raw %zu%s\n", name, bits, w, + bytes.size(), (w == bytes.size() ? " (content mismatch)" : "")); + ++g_failures; + return false; + } + // Sanity: every code indexes the dictionary. + for (uint16_t c : col.codes) { + if (c >= col.dict.num_tokens()) { + std::printf(" FAIL %-22s bits=%2u: code out of range\n", name, bits); + ++g_failures; + return false; + } + } + return true; +} + +std::vector SyntheticUrls(size_t n) { + const char* hosts[] = {"https://www.yandex.ru", "https://www.google.com", + "https://news.ycombinator.com", "http://m.yandex.ru", + "ftp://files.example.com"}; + const char* paths[] = {"/", "/search?q=", "/api/v1/data", "/blog/post-", "/users/"}; + std::vector out; + uint64_t x = 0x9E3779B97F4A7C15ull; + for (size_t i = 0; i < n; ++i) { + x += 0x9E3779B97F4A7C15ull; + std::string s = hosts[(x) % 5]; + s += paths[(x >> 16) % 5]; + s += std::to_string(static_cast(x >> 48)); + out.push_back(std::move(s)); + } + return out; +} + +} // namespace + +int main() { + std::printf("OnPair C++ port roundtrip tests\n"); + + for (uint8_t bits = 9; bits <= 16; ++bits) { + // Mixed lengths incl. empty, 1-byte, boundary 8/9, 16, >16. + Roundtrip({"", "a", "ab", "12345678", "123456789", "0123456789abcdef", + "0123456789abcdefGHIJ", "hello world hello world"}, + bits, "mixed_lengths"); + + // Binary with NUL bytes. + std::vector bin; + for (int i = 0; i < 40; ++i) { + std::string s; + for (int j = 0; j < 30; ++j) s.push_back(static_cast((i * 7 + j * 3) & 0xFF)); + bin.push_back(std::move(s)); + } + Roundtrip(bin, bits, "binary_nul"); + + // Homogeneous (heavy merges). + Roundtrip(std::vector(50, std::string(40, 'a')), bits, "homogeneous"); + + // Shared long prefix (exercises long bucket / trie promotion). + std::vector shared; + for (int i = 0; i < 300; ++i) shared.push_back("https://prefix/" + std::to_string(i)); + Roundtrip(shared, bits, "shared_long_prefix"); + + // Synthetic URLs. + Roundtrip(SyntheticUrls(20000), bits, "synthetic_urls"); + + // All-empty and single-empty edge cases. + Roundtrip({"", "", ""}, bits, "all_empty"); + Roundtrip({}, bits, "no_rows"); + } + + if (g_failures == 0) { + std::printf("ALL PASS\n"); + return 0; + } + std::printf("%d FAILURES\n", g_failures); + return 1; +} From ffe4a7fb9a1844a01bd2b0ddd20576616ecb2d1a Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 3 Aug 2026 06:02:06 +0000 Subject: [PATCH 18/24] Add common-prefix extraction for FSST and OnPair Splits each value into a shared prefix and a suffix before the symbol table sees it, following the FSST+ thesis (Alexandre, CWI 2025). Used to measure whether prefix extraction adds anything on top of either codec. --- cpp/src/parquet/onpair/prefix_plus.cc | 130 ++++++++++++++ cpp/src/parquet/onpair/prefix_plus.h | 76 ++++++++ .../onpair/prefix_plus_test_standalone.cc | 162 ++++++++++++++++++ 3 files changed, 368 insertions(+) create mode 100644 cpp/src/parquet/onpair/prefix_plus.cc create mode 100644 cpp/src/parquet/onpair/prefix_plus.h create mode 100644 cpp/src/parquet/onpair/prefix_plus_test_standalone.cc diff --git a/cpp/src/parquet/onpair/prefix_plus.cc b/cpp/src/parquet/onpair/prefix_plus.cc new file mode 100644 index 000000000000..b9f65ed321b0 --- /dev/null +++ b/cpp/src/parquet/onpair/prefix_plus.cc @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/onpair/prefix_plus.h" + +#include +#include +#include + +namespace parquet::prefix_plus { +namespace { + +// Longest common prefix of two strings, capped at `cap`. With `guard`, trim so +// the returned length never falls between an FSST escape byte (255) and its +// literal: if the last matched byte is 255, drop it (thesis Listing 5.4). +size_t Lcp(const uint8_t* a, size_t la, const uint8_t* b, size_t lb, size_t cap, bool guard) { + size_t m = std::min(std::min(la, lb), cap); + size_t l = 0; + while (l < m && a[l] == b[l]) ++l; + if (guard && l != 0 && a[l - 1] == 255) --l; + return l; +} + +// Thesis DP (sec 5.2.2) for one block: strings are strs[0..bn) / lens[0..bn) +// (already sorted). Writes prefix_len[k] and chunk_first_local[k] for each +// string k in the block; chunk_first_local is a block-local index. Returns the +// number of chunks that carry a non-empty prefix. +size_t CleaveBlock(const uint8_t* const* strs, const size_t* lens, size_t bn, size_t max_prefix, + bool guard, uint32_t* prefix_len, uint32_t* chunk_first_local) { + if (bn == 0) return 0; + + // Consecutive LCPs, then min_lcp[i][j] = shared prefix length of strings i..j + // as the running minimum of adjacent LCPs (standard range-LCP identity). + std::vector lcp(bn > 0 ? bn - 1 : 0); + for (size_t i = 0; i + 1 < bn; ++i) { + lcp[i] = Lcp(strs[i], lens[i], strs[i + 1], lens[i + 1], max_prefix, guard); + } + std::vector> min_lcp(bn, std::vector(bn, 0)); + for (size_t i = 0; i < bn; ++i) { + min_lcp[i][i] = static_cast(std::min(lens[i], max_prefix)); + for (size_t j = i + 1; j < bn; ++j) { + min_lcp[i][j] = std::min(min_lcp[i][j - 1], static_cast(lcp[j - 1])); + } + } + + std::vector len_prefix_sum(bn + 1, 0); + for (size_t i = 0; i < bn; ++i) len_prefix_sum[i + 1] = len_prefix_sum[i] + lens[i]; + + constexpr size_t kInf = std::numeric_limits::max(); + std::vector dp(bn + 1, kInf), prev(bn + 1, 0), pfx(bn + 1, 0); + dp[0] = 0; + for (size_t i = 1; i <= bn; ++i) { + for (size_t j = 0; j < i; ++j) { + if (dp[j] == kInf) continue; + const size_t mcp = min_lcp[j][i - 1]; + const size_t candidates[2] = {0, mcp}; + const int n_cand = mcp > 0 ? 2 : 1; + for (int c = 0; c < n_cand; ++c) { + const size_t p = candidates[c]; + const size_t cnt = i - j; + const size_t per_string_overhead = 1 + (p > 0 ? 2 : 0); // prefix_length [+ jumpback] + const size_t overhead = cnt * per_string_overhead; + const size_t sum_len = len_prefix_sum[i] - len_prefix_sum[j]; + // Store the shared prefix once (p bytes) + all suffixes (sum_len - cnt*p) + // + overhead == overhead + sum_len - (cnt-1)*p. + const size_t cost = dp[j] + overhead + sum_len - (cnt - 1) * p; + if (cost < dp[i]) { + dp[i] = cost; + prev[i] = j; + pfx[i] = p; + } + } + } + } + + // Backtrack into chunks (start_local, prefix_length), then assign per string. + size_t idx = bn; + size_t num_prefix_chunks = 0; + while (idx > 0) { + const size_t start = prev[idx]; + const size_t p = pfx[idx]; + if (p > 0) ++num_prefix_chunks; + for (size_t k = start; k < idx; ++k) { + prefix_len[k] = static_cast(p); + chunk_first_local[k] = static_cast(start); + } + idx = start; + } + return num_prefix_chunks; +} + +} // namespace + +Cleaving CleaveSorted(const uint8_t* const* strs, const size_t* lens, size_t n, size_t max_prefix, + bool guard_escape255) { + if (max_prefix > kMaxPrefix) max_prefix = kMaxPrefix; + Cleaving out; + out.prefix_len.assign(n, 0); + out.chunk_first.assign(n, 0); + out.num_prefix_chunks = 0; + + std::vector local_first(kBlockSize); + for (size_t base = 0; base < n; base += kBlockSize) { + const size_t bn = std::min(kBlockSize, n - base); + out.num_prefix_chunks += + CleaveBlock(strs + base, lens + base, bn, max_prefix, guard_escape255, + out.prefix_len.data() + base, local_first.data()); + // Lift block-local chunk-first indices to global positions. + for (size_t k = 0; k < bn; ++k) { + out.chunk_first[base + k] = static_cast(base) + local_first[k]; + } + } + return out; +} + +} // namespace parquet::prefix_plus diff --git a/cpp/src/parquet/onpair/prefix_plus.h b/cpp/src/parquet/onpair/prefix_plus.h new file mode 100644 index 000000000000..fbd03a6d32cf --- /dev/null +++ b/cpp/src/parquet/onpair/prefix_plus.h @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Common-prefix extraction ("+") for string codecs, the shared core behind +// FSST+ and OnPair+ in the benchmark harness. +// +// Algorithm: Y. L. Alexandre, "FSST+: Enhancing String Compression Through +// Common Prefix Extraction," MSc thesis, CWI, 2025. Within a sorted block of at +// most 128 strings, an optimal set of "similarity chunks" (ranges sharing a +// prefix) is chosen by dynamic programming (thesis sec 5.2.2): each shared +// prefix is stored once and every string keeps a 1-byte prefix length, an +// optional 2-byte jump-back offset, and its own suffix. The DP minimises total +// stored bytes and runs in O(B^2) per block (B = 128, constant), hence O(N) +// overall. +// +// This header exposes only the codec-agnostic cleaving decision. FSST+ runs it +// over FSST-compressed bytes (with `guard_escape255` so a prefix never splits an +// FSST escape from its literal); OnPair+ runs it over raw bytes. The two codecs' +// assembly and size accounting live in the benchmark, next to the other codecs. +// +// Little-endian hosts only. + +#pragma once + +#include +#include +#include + +namespace parquet::prefix_plus { + +/// Strings per block, and the window within which a prefix may be shared. The +/// thesis picks 128 (sec 5.4.1): a block fits in L1 and the O(B^2) DP stays cheap. +constexpr size_t kBlockSize = 128; + +/// Longest shareable prefix. Bounded by the u8 that stores a string's prefix +/// length in the block (thesis sec 3.1). +constexpr size_t kMaxPrefix = 255; + +/// The chosen cleaving for a whole sorted collection. Indices are positions in +/// the sorted collection (0..n). +struct Cleaving { + /// prefix_len[i] = bytes string i shares with (and borrows from) its chunk. + std::vector prefix_len; + /// chunk_first[i] = index of the first string of string i's chunk; its first + /// prefix_len[i] bytes are the shared prefix, stored once for the chunk. + std::vector chunk_first; + /// Number of similarity chunks with a non-empty shared prefix (diagnostic: + /// equals the count of prefixes actually stored). + size_t num_prefix_chunks = 0; +}; + +/// Run the thesis DP (sec 5.2.2) over the sorted collection `strs`/`lens` +/// (length n), independently per block of kBlockSize. `max_prefix` caps prefix +/// length (<= kMaxPrefix). When `guard_escape255` is set, a candidate prefix is +/// trimmed so it never ends between an FSST escape byte (255) and the literal it +/// escapes -- required when cleaving FSST-compressed bytes, and a no-op for raw +/// bytes. The collection must already be sorted so that strings sharing a prefix +/// are adjacent. +Cleaving CleaveSorted(const uint8_t* const* strs, const size_t* lens, size_t n, + size_t max_prefix, bool guard_escape255); + +} // namespace parquet::prefix_plus diff --git a/cpp/src/parquet/onpair/prefix_plus_test_standalone.cc b/cpp/src/parquet/onpair/prefix_plus_test_standalone.cc new file mode 100644 index 000000000000..f756eea7859b --- /dev/null +++ b/cpp/src/parquet/onpair/prefix_plus_test_standalone.cc @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Standalone validation for the prefix_plus cleaving core. No Arrow deps. +// Build: g++ -std=c++17 -O2 -Icpp/src \ +// cpp/src/parquet/onpair/prefix_plus.cc \ +// cpp/src/parquet/onpair/prefix_plus_test_standalone.cc -o t +// +// Checks the cleaving invariant that makes decode correct: for every string i, +// its first prefix_len[i] bytes equal the first prefix_len[i] bytes of its +// chunk representative chunk_first[i] (so reconstructing prefix++suffix rebuilds +// the string exactly), plus structural bounds and the FSST-escape guard. + +#include +#include +#include +#include +#include +#include + +#include "parquet/onpair/prefix_plus.h" + +namespace pp = parquet::prefix_plus; + +namespace { + +int g_failures = 0; + +// Sort `rows` (as the "+" codecs do before cleaving), run CleaveSorted, and +// verify the reconstruction invariant + bounds. `guard` mirrors the FSST-escape +// path. +bool Check(std::vector rows, bool guard, const char* name) { + std::sort(rows.begin(), rows.end()); + + std::vector ptrs(rows.size()); + std::vector lens(rows.size()); + for (size_t i = 0; i < rows.size(); ++i) { + ptrs[i] = reinterpret_cast(rows[i].data()); + lens[i] = rows[i].size(); + } + + pp::Cleaving cl = + pp::CleaveSorted(ptrs.data(), lens.data(), rows.size(), pp::kMaxPrefix, guard); + + if (cl.prefix_len.size() != rows.size() || cl.chunk_first.size() != rows.size()) { + std::printf(" FAIL %-22s: wrong result size\n", name); + ++g_failures; + return false; + } + + for (size_t i = 0; i < rows.size(); ++i) { + const uint32_t p = cl.prefix_len[i]; + const uint32_t cf = cl.chunk_first[i]; + // Bounds. + if (p > lens[i] || p > pp::kMaxPrefix || cf > i) { + std::printf(" FAIL %-22s: bad cleave at %zu (p=%u cf=%u len=%zu)\n", name, i, p, cf, + lens[i]); + ++g_failures; + return false; + } + // Same block (cleaving never crosses a 128-block boundary). + if (cf / pp::kBlockSize != i / pp::kBlockSize) { + std::printf(" FAIL %-22s: chunk crosses block at %zu\n", name, i); + ++g_failures; + return false; + } + // Reconstruct: prefix (from chunk rep) ++ suffix (own bytes after p) == row. + std::string rebuilt(rows[cf].data(), p); + rebuilt.append(rows[i].data() + p, lens[i] - p); + if (rebuilt != rows[i]) { + std::printf(" FAIL %-22s: reconstruction mismatch at %zu\n", name, i); + ++g_failures; + return false; + } + // FSST-escape guard: a prefix must not end right after an escape byte. + if (guard && p > 0 && static_cast(rows[i][p - 1]) == 255) { + std::printf(" FAIL %-22s: prefix splits escape at %zu\n", name, i); + ++g_failures; + return false; + } + } + return true; +} + +} // namespace + +int main() { + std::printf("prefix_plus cleaving tests\n"); + + // Identical strings (whole string becomes the shared prefix, empty suffixes). + Check(std::vector(200, "Customer#000000042"), false, "identical"); + + // Monotonic shared prefix (the target shape). + { + std::vector v; + for (int i = 1; i <= 500; ++i) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "Customer#%09d", i); + v.emplace_back(buf); + } + Check(v, false, "monotonic_prefix"); + } + + // No shared prefix (distinct first bytes) -> every prefix_len should be 0. + { + std::vector v; + for (int i = 0; i < 100; ++i) v.push_back(std::string(1, static_cast('A' + i % 26)) + + std::to_string(i)); + Check(v, false, "no_shared_prefix"); + } + + // Prefix longer than 255 must be capped (uint8 prefix_length). + Check(std::vector(64, std::string(400, 'z')), false, "over_255_cap"); + + // Empty and near-empty rows. + Check({"", "", "a", "ab", "abc"}, false, "empties"); + + // Multi-block (>128) with grouped prefixes across block boundaries. + { + std::vector v; + for (int g = 0; g < 10; ++g) + for (int i = 0; i < 40; ++i) v.push_back("group" + std::to_string(g) + "/item" + + std::to_string(i)); + Check(v, false, "multi_block"); + } + + // FSST-escape guard: rows containing byte 255 adjacent to shared regions. + { + std::vector v; + for (int i = 0; i < 60; ++i) { + std::string s = "pre"; + s.push_back(static_cast(255)); + s.push_back(static_cast('a' + i % 5)); + s += std::to_string(i); + v.push_back(std::move(s)); + } + Check(v, true, "escape_guard"); + } + + // Single row and empty input. + Check({"lonely"}, false, "single_row"); + Check({}, false, "no_rows"); + + if (g_failures == 0) { + std::printf("ALL PASS\n"); + return 0; + } + std::printf("%d FAILURES\n", g_failures); + return 1; +} From 87cc92e9792f4caf1e332e430a743b28d1af8fab Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 3 Aug 2026 06:02:06 +0000 Subject: [PATCH 19/24] Add the FSST-vs-OnPair benchmark and a corpora generator Measures FSST, OnPair, the prefix-extraction variants and zstd/lz4 pages on one corpus set, on three axes: compression ratio, whole-column decode with per-row random access, and encode. Every codec is charged a bit-packed per-row length array so column reconstruction costs the same across all of them. bench_common.h holds the timing and bit-packing helpers shared with the cascade benchmark. The Rust helper generates the corpora: TPC-H string columns, the OnPair paper's real-world datasets, ClickBench columns, and synthetic identifier and JSON sets. --- bench-fsst-onpair/.gitignore | 2 + bench-fsst-onpair/Cargo.lock | 919 +++++++++++++++ bench-fsst-onpair/Cargo.toml | 29 + bench-fsst-onpair/src/main.rs | 462 ++++++++ cpp/src/parquet/onpair/bench_common.h | 146 +++ .../parquet/onpair/fsst_onpair_benchmark.cc | 1036 +++++++++++++++++ 6 files changed, 2594 insertions(+) create mode 100644 bench-fsst-onpair/.gitignore create mode 100644 bench-fsst-onpair/Cargo.lock create mode 100644 bench-fsst-onpair/Cargo.toml create mode 100644 bench-fsst-onpair/src/main.rs create mode 100644 cpp/src/parquet/onpair/bench_common.h create mode 100644 cpp/src/parquet/onpair/fsst_onpair_benchmark.cc diff --git a/bench-fsst-onpair/.gitignore b/bench-fsst-onpair/.gitignore new file mode 100644 index 000000000000..338fa47820d5 --- /dev/null +++ b/bench-fsst-onpair/.gitignore @@ -0,0 +1,2 @@ +/target +/corpora diff --git a/bench-fsst-onpair/Cargo.lock b/bench-fsst-onpair/Cargo.lock new file mode 100644 index 000000000000..e1c8fe5f0143 --- /dev/null +++ b/bench-fsst-onpair/Cargo.lock @@ -0,0 +1,919 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arrow" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ord" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" + +[[package]] +name = "arrow-select" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "59.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bench-fsst-onpair" +version = "0.1.0" +dependencies = [ + "arrow-array", + "arrow-schema", + "fsst-rs", + "onpair", + "tpchgen", + "tpchgen-arrow", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "fsst-rs" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b13ac798afc0d9194eb4efefef8b9332efbd80b43f302a968cb8cb23b9d5360" +dependencies = [ + "rustc-hash", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "onpair" +version = "0.1.1" +source = "git+https://github.com/spiraldb/onpair?rev=f5f4bfe1fb7221c42a9f8261c269bdf5bd31052b#f5f4bfe1fb7221c42a9f8261c269bdf5bd31052b" +dependencies = [ + "hashbrown 0.16.1", + "rand", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tpchgen" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e26c6d047694183517044ea72d2e0337b05b43d40809f0958f603a7e5a286855" + +[[package]] +name = "tpchgen-arrow" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64c19d707b783f219883549268da8ea98f91e7aaec3b626563044a21c2b37aa" +dependencies = [ + "arrow", + "tpchgen", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/bench-fsst-onpair/Cargo.toml b/bench-fsst-onpair/Cargo.toml new file mode 100644 index 000000000000..c7f19c6f446d --- /dev/null +++ b/bench-fsst-onpair/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "bench-fsst-onpair" +version = "0.1.0" +edition = "2024" +publish = false + +[[bin]] +name = "bench-fsst-onpair" +path = "src/main.rs" + +[dependencies] +# A published OnPair implementation (the algorithm of arXiv:2508.02280), pinned +# for reproducibility. Git dep => Cargo passes --cap-lints allow, so its +# deny(warnings) won't fail our build on a newer rustc. +onpair = { git = "https://github.com/spiraldb/onpair", rev = "f5f4bfe1fb7221c42a9f8261c269bdf5bd31052b" } + +# FSST reference codec (same algorithm as the Arrow C++ PR #48232 vendors). +fsst-rs = "0.5.11" + +# In-process TPC-H data generation (no external files needed). +tpchgen = "3.0.0" +tpchgen-arrow = "3.0.0" +arrow-array = "59.1" +arrow-schema = "59.1" + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 diff --git a/bench-fsst-onpair/src/main.rs b/bench-fsst-onpair/src/main.rs new file mode 100644 index 000000000000..7d7e6ef06223 --- /dev/null +++ b/bench-fsst-onpair/src/main.rs @@ -0,0 +1,462 @@ +//! Apples-to-apples comparison of FSST vs OnPair16 (the paper's 16-byte-max-token +//! variant, arXiv:2508.02280) at 12- and 16-bit dictionary sizes, on the same +//! string corpora. +//! +//! Both codecs run in a single Rust process so encode/decode throughput is +//! measured under one harness. Pin to a single core with `taskset -c 0`. +//! +//! Corpora: +//! * TPC-H string columns (o_comment, p_name, l_comment, c_comment), +//! generated in-process via tpchgen at scale factor 1. +//! * ClickBench: real `hits.parquet`-style data if ONPAIR_BENCH_PARQUET is +//! set (+ optional ONPAIR_BENCH_COLUMN), else a synthetic URL corpus. +//! +//! Size accounting (raw codec output, no downstream integer compression): +//! * OnPair = dict bytes + dict offsets(u32) + codes(u16) + row offsets(u32) +//! * FSST = symbol table + symbol lengths + code bytes + row offsets(u32) +//! +//! Both count an (n+1) u32 row-offset vector so the comparison is fair. + +use std::hint::black_box; +use std::mem::MaybeUninit; +use std::time::Instant; + +use arrow_array::cast::AsArray; +use fsst::Compressor; +use onpair::{Config, MaxDictBits, Threshold, compress as onpair_compress}; +use tpchgen::generators::{ + CustomerGenerator, LineItemGenerator, OrderGenerator, PartGenerator, SupplierGenerator, +}; +use tpchgen_arrow::{ + CustomerArrow, LineItemArrow, OrderArrow, PartArrow, RecordBatchIterator, SupplierArrow, +}; + +const BATCH_SIZE: usize = 8192 * 8; +/// Every corpus is truncated/generated to exactly this many rows for a fair +/// equal-N comparison. +const TARGET_ROWS: usize = 500_000; +const ENCODE_ITERS: usize = 3; +const DECODE_ITERS: usize = 10; + +/// A packed string corpus: concatenated bytes + (n+1) u64 offsets. +struct Corpus { + name: String, + bytes: Vec, + offsets: Vec, + /// Per-row slices, precomputed for FSST training/compression. + n_rows: usize, +} + +impl Corpus { + fn new(name: impl Into, bytes: Vec, offsets: Vec) -> Self { + let n_rows = offsets.len() - 1; + Corpus { name: name.into(), bytes, offsets, n_rows } + } + fn raw_bytes(&self) -> usize { + self.bytes.len() + } + fn rows(&self) -> Vec<&[u8]> { + (0..self.n_rows) + .map(|i| &self.bytes[self.offsets[i] as usize..self.offsets[i + 1] as usize]) + .collect() + } +} + +fn median(mut v: Vec) -> f64 { + v.sort_by(|a, b| a.partial_cmp(b).unwrap()); + v[v.len() / 2] +} + +fn mib(bytes: usize) -> f64 { + bytes as f64 / (1024.0 * 1024.0) +} + +// ───────────────────────────── OnPair ───────────────────────────── + +struct Measured { + label: String, + compressed_bytes: usize, + encode_mibs: f64, + decode_mibs: f64, +} + +fn run_onpair(c: &Corpus, bits: u8, threshold: f64) -> Measured { + let cfg = Config { + max_dict_bits: MaxDictBits::new(bits).unwrap(), + threshold: Threshold::new(threshold).unwrap(), + seed: Some(42), + }; + // Compressed size + a live column for decode timing. + let col = onpair_compress(&c.bytes, &c.offsets, cfg).unwrap(); + let compressed = col.dict.bytes().len() + + col.dict.offsets().len() * 4 + + col.codes.len() * 2 + + col.row_offsets.len() * 4; + + // Encode throughput: full train + compress, median of ENCODE_ITERS. + let mut enc = Vec::with_capacity(ENCODE_ITERS); + for _ in 0..ENCODE_ITERS { + let t = Instant::now(); + let out = onpair_compress(black_box(&c.bytes), black_box(&c.offsets), cfg).unwrap(); + let dt = t.elapsed().as_secs_f64(); + black_box(&out); + enc.push(mib(c.raw_bytes()) / dt); + } + + // Decode throughput: whole-column decompress_into. + let cap = col.view().decoded_len() + 16; // + tail padding + // Correctness: whole-column decode must reconstruct the concatenated input. + { + let mut buf: Vec> = vec![MaybeUninit::uninit(); cap]; + let n = unsafe { col.view().decompress_into(&mut buf) }; + let decoded: &[u8] = unsafe { std::slice::from_raw_parts(buf.as_ptr() as *const u8, n) }; + assert_eq!(n, c.raw_bytes(), "OnPair{bits} decoded len mismatch on {}", c.name); + assert!(decoded == c.bytes.as_slice(), "OnPair{bits} roundtrip mismatch on {}", c.name); + } + let mut dec = Vec::with_capacity(DECODE_ITERS); + for _ in 0..DECODE_ITERS { + let mut buf: Vec> = vec![MaybeUninit::uninit(); cap]; + let t = Instant::now(); + // SAFETY: buf sized to decoded_len()+padding; view from a trusted column. + let n = unsafe { col.view().decompress_into(&mut buf) }; + let dt = t.elapsed().as_secs_f64(); + black_box(&buf[..n]); + dec.push(mib(c.raw_bytes()) / dt); + } + + Measured { + label: format!("OnPair{bits}"), + compressed_bytes: compressed, + encode_mibs: median(enc), + decode_mibs: median(dec), + } +} + +// ───────────────────────────── FSST ───────────────────────────── + +/// Train + compress every row into one concatenated code buffer with +/// (n+1) u32 offsets. Returns (compressor, codes, offsets). +fn fsst_encode(rows: &Vec<&[u8]>) -> (Compressor, Vec, Vec) { + let compressor = Compressor::train(rows); + let total: usize = rows.iter().map(|r| r.len()).sum(); + let mut codes: Vec = Vec::with_capacity(2 * total + 8 * rows.len() + 16); + let mut offsets: Vec = Vec::with_capacity(rows.len() + 1); + offsets.push(0); + let mut scratch: Vec = Vec::with_capacity(1024); + for r in rows { + scratch.clear(); + // FSST worst case: 2 bytes per input byte + a few for escapes. + let need = 2 * r.len() + 16; + if scratch.capacity() < need { + scratch.reserve(need - scratch.capacity()); + } + // SAFETY: scratch has capacity for the FSST worst-case output of `r`. + unsafe { compressor.compress_into(r, &mut scratch) }; + codes.extend_from_slice(&scratch); + offsets.push(codes.len() as u32); + } + (compressor, codes, offsets) +} + +fn run_fsst(c: &Corpus) -> Measured { + let rows = c.rows(); + + let (compressor, codes, offsets) = fsst_encode(&rows); + let compressed = std::mem::size_of_val(compressor.symbol_table()) + + std::mem::size_of_val(compressor.symbol_lengths()) + + codes.len() + + offsets.len() * 4; + + // Encode throughput. + let mut enc = Vec::with_capacity(ENCODE_ITERS); + for _ in 0..ENCODE_ITERS { + let t = Instant::now(); + let out = fsst_encode(black_box(&rows)); + let dt = t.elapsed().as_secs_f64(); + black_box(&out); + enc.push(mib(c.raw_bytes()) / dt); + } + + // Decode throughput: decompress the whole concatenated code stream at once + // (FSST codes are context-free, so this reconstructs concatenated plaintext). + let decompressor = compressor.decompressor(); + let cap = c.raw_bytes() + 16; + // Correctness: decoding the concatenated code stream reconstructs the input. + { + let mut buf: Vec> = vec![MaybeUninit::uninit(); cap]; + let n = decompressor.decompress_into(&codes, &mut buf); + let decoded: &[u8] = unsafe { std::slice::from_raw_parts(buf.as_ptr() as *const u8, n) }; + assert_eq!(n, c.raw_bytes(), "FSST decoded len mismatch on {}", c.name); + assert!(decoded == c.bytes.as_slice(), "FSST roundtrip mismatch on {}", c.name); + } + let mut dec = Vec::with_capacity(DECODE_ITERS); + for _ in 0..DECODE_ITERS { + let mut buf: Vec> = vec![MaybeUninit::uninit(); cap]; + let t = Instant::now(); + let n = decompressor.decompress_into(black_box(&codes), &mut buf); + let dt = t.elapsed().as_secs_f64(); + black_box(&buf[..n]); + dec.push(mib(c.raw_bytes()) / dt); + } + + Measured { + label: "FSST".to_string(), + compressed_bytes: compressed, + encode_mibs: median(enc), + decode_mibs: median(dec), + } +} + +// ───────────────────────────── Corpora ───────────────────────────── + +/// Load any TPC-H string column, dispatching to its table generator by the +/// column-name prefix (o_/l_/c_/p_/s_). +fn tpch_column(col: &str) -> Corpus { + // Scale factor per table so it yields >= TARGET_ROWS rows (rows/SF at SF1: + // lineitem 6.0M, orders 1.5M, customer 150k, part 200k, supplier 10k), then + // truncate to exactly TARGET_ROWS. + let sf: f64 = match col.split('_').next().unwrap() { + "l" | "o" => 1.0, + "c" => 4.0, // 150k * 4 = 600k + "p" => 3.0, // 200k * 3 = 600k + "s" => 50.0, // 10k * 50 = 500k + _ => 1.0, + }; + let idx_of = |schema: &arrow_schema::Schema| { + schema.fields().iter().position(|f| f.name() == col).unwrap_or_else(|| { + panic!("column {col} not found in table schema") + }) + }; + let (bytes, offsets) = match col.split('_').next().unwrap() { + "l" => { + let it = LineItemArrow::new(LineItemGenerator::new(sf, 1, 1)).with_batch_size(BATCH_SIZE); + let schema = it.schema().clone(); + collect(it, idx_of(&schema)) + } + "o" => { + let it = OrderArrow::new(OrderGenerator::new(sf, 1, 1)).with_batch_size(BATCH_SIZE); + let schema = it.schema().clone(); + collect(it, idx_of(&schema)) + } + "c" => { + let it = CustomerArrow::new(CustomerGenerator::new(sf, 1, 1)).with_batch_size(BATCH_SIZE); + let schema = it.schema().clone(); + collect(it, idx_of(&schema)) + } + "p" => { + let it = PartArrow::new(PartGenerator::new(sf, 1, 1)).with_batch_size(BATCH_SIZE); + let schema = it.schema().clone(); + collect(it, idx_of(&schema)) + } + "s" => { + let it = SupplierArrow::new(SupplierGenerator::new(sf, 1, 1)).with_batch_size(BATCH_SIZE); + let schema = it.schema().clone(); + collect(it, idx_of(&schema)) + } + other => panic!("unknown table prefix {other} for column {col}"), + }; + Corpus::new(format!("tpch/{col}"), bytes, offsets) +} + +fn collect(batches: I, idx: usize) -> (Vec, Vec) +where + I: Iterator, +{ + let mut bytes = Vec::new(); + let mut offsets: Vec = vec![0]; + 'outer: for batch in batches { + let arr = batch.column(idx).as_string_view(); + for v in arr.iter() { + let s = v.unwrap_or("").as_bytes(); + bytes.extend_from_slice(s); + offsets.push(bytes.len() as u64); + if offsets.len() > TARGET_ROWS { + break 'outer; + } + } + } + (bytes, offsets) +} + +fn clickbench_corpus() -> Corpus { + if let Ok(path) = std::env::var("ONPAIR_BENCH_PARQUET") { + if let Some((bytes, offsets, colname)) = read_parquet(&path) { + return Corpus::new(format!("clickbench/{colname}"), bytes, offsets); + } + eprintln!("warning: could not read {path}, falling back to synthetic"); + } + let (bytes, offsets) = synthetic_clickbench_urls(TARGET_ROWS); + Corpus::new("clickbench/synthetic-urls", bytes, offsets) +} + +fn read_parquet(_path: &str) -> Option<(Vec, Vec, String)> { + // Only wired when ONPAIR_BENCH_PARQUET is set; requires the `parquet` crate. + // Left unimplemented to keep the default build light; synthetic is used. + None +} + +fn synthetic_clickbench_urls(n: usize) -> (Vec, Vec) { + const HOSTS: &[&str] = &[ + "https://www.yandex.ru", "https://www.google.com", "https://news.ycombinator.com", + "https://www.example.com", "https://docs.example.org", "https://api.example.net", + "http://m.yandex.ru", "https://maps.example.com", "https://shop.example.com", + "ftp://files.example.com", + ]; + const PATHS: &[&str] = &[ + "/", "/page", "/news", "/search?q=", "/profile", "/login", "/api/v1/data", + "/static/asset.png", "/blog/post-", "/feed.xml", "/sitemap.xml", "/users/", + "/admin/dashboard", "/categories/electronics", "/cart/checkout", + ]; + const TAILS: &[&str] = &["", "alpha", "beta", "gamma", "delta", "001", "002", "003"]; + let mut bytes = Vec::new(); + let mut offsets: Vec = vec![0]; + let mut x = 0x9E3779B97F4A7C15u64; + for _ in 0..n { + x = x.wrapping_add(0x9E3779B97F4A7C15); + let h = HOSTS[(x as usize) % HOSTS.len()]; + let p = PATHS[((x >> 16) as usize) % PATHS.len()]; + let t = TAILS[((x >> 32) as usize) % TAILS.len()]; + let num = (x >> 48) as u16; + let s = format!("{h}{p}{t}{num}"); + bytes.extend_from_slice(s.as_bytes()); + offsets.push(bytes.len() as u64); + } + (bytes, offsets) +} + +/// Synthetic JSON "event" objects: a fixed schema (repeated keys + structural +/// punctuation) with per-row varying values. High-cardinality overall (unique +/// ids/timestamps), but saturated with shared <=16-byte fragments — the shape a +/// Variant/JSON column takes. Newline-free per row. +fn synthetic_variant_json(n: usize) -> (Vec, Vec) { + const EVENTS: &[&str] = &["click", "view", "purchase", "signup", "logout", "search"]; + const COUNTRIES: &[&str] = &["US", "GB", "DE", "FR", "JP", "IN", "BR", "CA"]; + const TIERS: &[&str] = &["free", "pro", "enterprise"]; + let mut bytes = Vec::new(); + let mut offsets: Vec = vec![0]; + let mut x = 0x1234_5678_9ABC_DEF0u64; + let mut next = || { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x + }; + for _ in 0..n { + let uid = next() % 10_000_000; + let ev = EVENTS[(next() as usize) % EVENTS.len()]; + let country = COUNTRIES[(next() as usize) % COUNTRIES.len()]; + let tier = TIERS[(next() as usize) % TIERS.len()]; + let hh = next() % 24; + let mm = next() % 60; + let ss = next() % 60; + let day = 1 + next() % 28; + let amount = (next() % 100000) as f64 / 100.0; + let session = next(); + let s = format!( + "{{\"user_id\":{uid},\"event\":\"{ev}\",\"ts\":\"2024-03-{day:02}T{hh:02}:{mm:02}:{ss:02}Z\",\ +\"country\":\"{country}\",\"tier\":\"{tier}\",\"amount\":{amount:.2},\"session\":\"{session:016x}\"}}" + ); + bytes.extend_from_slice(s.as_bytes()); + offsets.push(bytes.len() as u64); + } + (bytes, offsets) +} + +// ───────────────────────────── main ───────────────────────────── + +/// Write each corpus as a newline-delimited .txt (one row per line) into `dir`, +/// so the C++ harness reads byte-identical inputs. All these columns are +/// newline-free (TPC-H comments/names, synthetic URLs), so line-delimiting is +/// lossless here. +fn dump_corpora(dir: &str) { + use std::io::Write; + std::fs::create_dir_all(dir).expect("create dump dir"); + // A spread across data shapes: free text, multi-word names/types, + // high-cardinality addresses, patterned IDs, and low-cardinality enums. + let tpch_cols = [ + // free text + "o_comment", "l_comment", "c_comment", "p_comment", "s_comment", + // multi-word names / types + "p_name", "p_type", "c_name", "s_name", + // high-cardinality addresses + "c_address", "s_address", + // patterned IDs / numbers + "o_clerk", "c_phone", + // low-cardinality enums / small vocab + "o_orderpriority", "l_shipmode", "p_brand", "p_container", "c_mktsegment", + ]; + let mut corpora: Vec = tpch_cols.iter().map(|c| tpch_column(c)).collect(); + corpora.push(clickbench_corpus()); + { + let (bytes, offsets) = synthetic_variant_json(TARGET_ROWS); + corpora.push(Corpus::new("variant/json-events", bytes, offsets)); + } + for c in &corpora { + let fname = c.name.replace('/', "_"); + let path = format!("{dir}/{fname}.txt"); + let f = std::fs::File::create(&path).expect("create file"); + let mut w = std::io::BufWriter::new(f); + for i in 0..c.n_rows { + let s = &c.bytes[c.offsets[i] as usize..c.offsets[i + 1] as usize]; + assert!(!s.contains(&b'\n'), "row contains newline in {}", c.name); + w.write_all(s).unwrap(); + w.write_all(b"\n").unwrap(); + } + w.flush().unwrap(); + eprintln!("[dump] {path}: {} rows, {:.2} MiB", c.n_rows, mib(c.raw_bytes())); + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + if let Some(pos) = args.iter().position(|a| a == "--dump-corpora") { + let dir = args.get(pos + 1).map(String::as_str).unwrap_or("corpora"); + dump_corpora(dir); + return; + } + + // TPC-H uses threshold 0.2 (matches onpair's tpch bench); ClickBench 0.5. + let corpora: Vec<(Corpus, f64)> = vec![ + (tpch_column("o_comment"), 0.2), + (tpch_column("l_comment"), 0.2), + (tpch_column("c_comment"), 0.2), + (tpch_column("p_name"), 0.2), + (clickbench_corpus(), 0.5), + ]; + + println!( + "{:<26} {:>10} {:>10} {:>9} {:>9} {:>9} {:>9} {:>9} {:>9}", + "corpus", "rows", "raw MiB", + "ratio", "enc MiB/s", "dec MiB/s", "", "", "" + ); + println!("{}", "─".repeat(120)); + + for (c, threshold) in &corpora { + let fsst = run_fsst(c); + let op12 = run_onpair(c, 12, *threshold); + let op16 = run_onpair(c, 16, *threshold); + + println!( + "{:<26} {:>10} {:>10.2}", + c.name, c.n_rows, mib(c.raw_bytes()) + ); + for m in [&fsst, &op12, &op16] { + let ratio = c.raw_bytes() as f64 / m.compressed_bytes as f64; + println!( + " {:<24} {:>10} {:>10.2} {:>8.3}x {:>9.1} {:>9.1}", + m.label, "", mib(m.compressed_bytes), + ratio, m.encode_mibs, m.decode_mibs + ); + } + // Head-to-head deltas: OnPair16 vs FSST. + let ratio_fsst = c.raw_bytes() as f64 / fsst.compressed_bytes as f64; + let ratio_op16 = c.raw_bytes() as f64 / op16.compressed_bytes as f64; + println!( + " → OnPair16 vs FSST: ratio {:+.1}%, encode {:+.1}%, decode {:+.1}%", + (ratio_op16 / ratio_fsst - 1.0) * 100.0, + (op16.encode_mibs / fsst.encode_mibs - 1.0) * 100.0, + (op16.decode_mibs / fsst.decode_mibs - 1.0) * 100.0, + ); + println!(); + } +} diff --git a/cpp/src/parquet/onpair/bench_common.h b/cpp/src/parquet/onpair/bench_common.h new file mode 100644 index 000000000000..f8d1b4e3bbb0 --- /dev/null +++ b/cpp/src/parquet/onpair/bench_common.h @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Pieces shared by the standalone string-encoding benchmarks: the packed corpus +// representation, the size-accounting helpers, and the timing conventions. +// +// This header exists so that fsst_onpair_benchmark.cc (FSST / zstd / lz4 / +// OnPair, no Arrow dependency) and cascade_benchmark.cc (the same codecs plus +// Parquet's own byte-array encodings, which needs libparquet) cannot drift apart +// on how bytes are counted. A ratio is only comparable across the two binaries +// if every codec in both is charged the same way, so the accounting lives here +// and nowhere else. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +// zstd: declare the small, ABI-stable subset we use so the standalone build +// needs only the installed libzstd (no dev header). Link libzstd.so directly. +extern "C" { +size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int level); +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +size_t ZSTD_compressBound(size_t srcSize); +unsigned ZSTD_isError(size_t code); +// lz4 (fast block compressor) - ABI-stable subset; link the installed liblz4. +int LZ4_compressBound(int inputSize); +int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); +int LZ4_decompress_safe(const char* src, char* dst, int compressedSize, int dstCapacity); +} + +namespace bench { + +using Clock = std::chrono::steady_clock; + +constexpr int kEncodeIters = 3; +constexpr int kDecodeIters = 10; + +inline double Mib(size_t bytes) { return static_cast(bytes) / (1024.0 * 1024.0); } + +inline double Median(std::vector v) { + std::sort(v.begin(), v.end()); + return v[v.size() / 2]; +} + +// Bits to store a value in [0, x] (x==0 -> 0 bits). +inline size_t BitWidth(uint64_t x) { + return x == 0 ? 0 : 64 - static_cast(__builtin_clzll(x)); +} +// Bits to index `count` distinct symbols [0, count) (== ceil(log2 count), >=1). +inline size_t IndexBits(size_t count) { + return count <= 1 ? 1 + : (64 - static_cast( + __builtin_clzll(static_cast(count - 1)))); +} +inline size_t BitPackedBytes(size_t n, size_t bits) { return (n * bits + 7) / 8; } + +// A packed corpus: concatenated bytes + (n+1) u32 offsets. +struct Corpus { + std::string name; + std::vector bytes; + std::vector offsets; + size_t n_rows() const { return offsets.size() - 1; } + size_t raw_bytes() const { return bytes.size(); } + size_t max_row_len() const { + size_t m = 0; + for (size_t i = 0; i < n_rows(); ++i) m = std::max(m, offsets[i + 1] - offsets[i]); + return m; + } + // Realistic per-row row-length side array (delta offsets), bit-packed at the + // width of the longest row. Charged to every value-preserving codec (FSST, + // zstd, lz4, OnPair) so the row boundaries are accounted the way a real + // columnar format stores them - not as raw (n+1) u32. + // + // NOT charged to Parquet's own byte-array encodings: PLAIN and the DELTA_* + // family embed their lengths in the encoded payload, so adding this on top + // would count row boundaries twice. + size_t len_array_bytes() const { + return BitPackedBytes(n_rows(), std::max(1, BitWidth(max_row_len()))); + } +}; + +// Read a newline-delimited file (one row per line) into a packed corpus. +inline Corpus ReadCorpus(const std::filesystem::path& path) { + Corpus c; + c.name = path.stem().string(); + std::ifstream in(path, std::ios::binary); + c.offsets.push_back(0); + std::string line; + while (std::getline(in, line)) { + c.bytes.insert(c.bytes.end(), line.begin(), line.end()); + c.offsets.push_back(static_cast(c.bytes.size())); + } + return c; +} + +struct Measured { + std::string label; + size_t compressed_bytes = 0; + double encode_mibs = 0; + double decode_mibs = 0; +}; + +// TPC-H columns train with a 0.2 sample fraction; the URL corpus with 0.5 +// (matching the Rust harness). +inline double ThresholdFor(const std::string& name) { + return name.rfind("tpch_", 0) == 0 ? 0.2 : 0.5; +} + +// Collect the .txt corpora in `dir`, sorted, so both binaries iterate the same +// set in the same order. +inline std::vector CorpusFiles(const std::string& dir) { + std::vector files; + for (const auto& e : std::filesystem::directory_iterator(dir)) { + if (e.path().extension() == ".txt") files.push_back(e.path()); + } + std::sort(files.begin(), files.end()); + return files; +} + +// Resolve the corpus directory: argv[1], then $ONPAIR_BENCH_DIR, then the +// generator's default output path. +inline std::string CorpusDir(int argc, char** argv) { + if (argc > 1) return argv[1]; + if (const char* env = std::getenv("ONPAIR_BENCH_DIR")) return env; + return "bench-fsst-onpair/corpora"; +} + +} // namespace bench diff --git a/cpp/src/parquet/onpair/fsst_onpair_benchmark.cc b/cpp/src/parquet/onpair/fsst_onpair_benchmark.cc new file mode 100644 index 000000000000..50c62f5bda9c --- /dev/null +++ b/cpp/src/parquet/onpair/fsst_onpair_benchmark.cc @@ -0,0 +1,1036 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Standalone C++ comparison of FSST (Arrow PR #48232 vendored codec), a C++ +// OnPair implementation, zstd level 1 and lz4, in one process on identical +// string corpora. Reports compression ratio and encode/decode throughput. +// Corpora are produced by the Rust bench-fsst-onpair/ generator (--dump-corpora). +// +// All codecs run single-threaded; pin the process with `taskset -c 0`, and run +// it twice using the warm second run. +// +// Build (from the Arrow repo root), one line: +// g++ -std=c++17 -O3 -march=native -Icpp/src -Icpp/thirdparty/fsst +// cpp/thirdparty/fsst/libfsst.cpp cpp/thirdparty/fsst/fsst_avx512.cpp +// cpp/src/parquet/onpair/onpair.cc cpp/src/parquet/onpair/prefix_plus.cc +// cpp/src/parquet/onpair/fsst_onpair_benchmark.cc +// /usr/lib64/libzstd.so.1 /usr/lib64/liblz4.so.1 -o /tmp/fsst_onpair_bench +// +// Run: taskset -c 0 /tmp/fsst_onpair_bench +// (corpora_dir defaults to $ONPAIR_BENCH_DIR, then ./bench-fsst-onpair/corpora) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fsst.h" +#include "parquet/onpair/bench_common.h" +#include "parquet/onpair/onpair.h" +#include "parquet/onpair/prefix_plus.h" + +namespace op = parquet::onpair; + +namespace { + +// Corpus / Measured / Mib / Median / BitWidth / IndexBits / BitPackedBytes / +// Clock / the iteration counts / ThresholdFor - shared with cascade_benchmark.cc +// so both binaries account bytes identically. +using namespace bench; // NOLINT(build/namespaces) + +// FSST + +// Compress the whole corpus into one packed buffer; returns compressed bytes, +// per-row lengths, and the serialized symbol-table size. +struct FsstEncoded { + std::vector output; // packed compressed bytes + size_t total = 0; // used bytes in `output` + size_t table_bytes = 0; // fsst_export size (symbol table) +}; + +FsstEncoded FsstEncode(const Corpus& c) { + size_t n = c.n_rows(); + std::vector lenIn(n); + std::vector strIn(n); + for (size_t i = 0; i < n; ++i) { + lenIn[i] = c.offsets[i + 1] - c.offsets[i]; + strIn[i] = c.bytes.data() + c.offsets[i]; + } + fsst_encoder_t* enc = fsst_create(n, lenIn.data(), strIn.data(), 0); + + // Conservative per-string bound from fsst.h: 7 + 2*len. + size_t out_cap = 7 * n + 2 * c.raw_bytes() + 16; + FsstEncoded e; + e.output.resize(out_cap); + std::vector lenOut(n); + std::vector strOut(n); + size_t done = fsst_compress(enc, n, lenIn.data(), strIn.data(), out_cap, e.output.data(), + lenOut.data(), strOut.data()); + if (done != n) { + std::fprintf(stderr, "FSST: only compressed %zu/%zu rows\n", done, n); + std::abort(); + } + e.total = 0; + for (size_t i = 0; i < n; ++i) e.total += lenOut[i]; + + unsigned char table[FSST_MAXHEADER]; + e.table_bytes = fsst_export(enc, table); + fsst_destroy(enc); + return e; +} + +Measured RunFsst(const Corpus& c) { + size_t n = c.n_rows(); + FsstEncoded e = FsstEncode(c); + Measured m; + m.label = "FSST"; + m.compressed_bytes = e.table_bytes + e.total + c.len_array_bytes(); + + // Encode throughput (train + compress). + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + FsstEncoded tmp = FsstEncode(c); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(tmp.total) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Rebuild a decoder from the packed stream for decode timing. + std::vector lenIn(n); + std::vector strIn(n); + for (size_t i = 0; i < n; ++i) { + lenIn[i] = c.offsets[i + 1] - c.offsets[i]; + strIn[i] = c.bytes.data() + c.offsets[i]; + } + fsst_encoder_t* enc2 = fsst_create(n, lenIn.data(), strIn.data(), 0); + fsst_decoder_t dec = fsst_decoder(enc2); + fsst_destroy(enc2); + + size_t cap = c.raw_bytes() + 16; + // Correctness: decoding the whole packed stream reconstructs the input. + { + std::vector out(cap); + size_t w = fsst_decompress(&dec, e.total, e.output.data(), cap, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "FSST roundtrip mismatch on %s (w=%zu raw=%zu)\n", c.name.c_str(), w, + c.raw_bytes()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap); + auto t0 = Clock::now(); + size_t w = fsst_decompress(&dec, e.total, e.output.data(), cap, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// zstd (block-compression baseline; no random access) + +// Compresses the whole concatenated corpus in one frame at the given level. +// Unlike FSST/OnPair this gives no per-row random access - it's a reference for +// what a general-purpose block compressor achieves on the same bytes. +Measured RunZstd(const Corpus& c, int level) { + size_t bound = ZSTD_compressBound(c.raw_bytes()); + std::vector comp(bound); + size_t csize = ZSTD_compress(comp.data(), bound, c.bytes.data(), c.raw_bytes(), level); + if (ZSTD_isError(csize)) { + std::fprintf(stderr, "zstd compress error on %s\n", c.name.c_str()); + std::abort(); + } + Measured m; + m.label = "zstd(" + std::to_string(level) + ")"; + // Add (n+1) u32 row offsets so row recovery is accounted for, as with the + // other codecs (zstd's frame decompresses to concatenated plaintext only). + m.compressed_bytes = csize + c.len_array_bytes(); + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + size_t r = ZSTD_compress(comp.data(), bound, c.bytes.data(), c.raw_bytes(), level); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(r) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + size_t cap = c.raw_bytes() + 16; + { + std::vector out(cap); + size_t w = ZSTD_decompress(out.data(), cap, comp.data(), csize); + if (ZSTD_isError(w) || w != c.raw_bytes() || + std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "zstd roundtrip mismatch on %s\n", c.name.c_str()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap); + auto t0 = Clock::now(); + size_t w = ZSTD_decompress(out.data(), cap, comp.data(), csize); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// lz4 (fast block-compression baseline; no random access) + +Measured RunLz4(const Corpus& c) { + int raw = static_cast(c.raw_bytes()); + int bound = LZ4_compressBound(raw); + std::vector comp(bound); + int csize = LZ4_compress_default(reinterpret_cast(c.bytes.data()), comp.data(), raw, + bound); + if (csize <= 0) { + std::fprintf(stderr, "lz4 compress error on %s\n", c.name.c_str()); + std::abort(); + } + Measured m; + m.label = "lz4"; + m.compressed_bytes = static_cast(csize) + c.len_array_bytes(); // + bit-packed lengths + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + int r = LZ4_compress_default(reinterpret_cast(c.bytes.data()), comp.data(), raw, + bound); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(r) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + int cap = raw + 16; + { + std::vector out(cap); + int w = LZ4_decompress_safe(comp.data(), out.data(), csize, cap); + if (w != raw || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "lz4 roundtrip mismatch on %s\n", c.name.c_str()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap); + auto t0 = Clock::now(); + int w = LZ4_decompress_safe(comp.data(), out.data(), csize, cap); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// OnPair + +Measured RunOnPair(const Corpus& c, uint8_t bits, double threshold) { + op::Config cfg; + cfg.max_dict_bits = bits; + cfg.threshold_fraction = threshold; + cfg.seed = 42; + size_t n = c.n_rows(); + + op::Column col = op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), n, cfg); + Measured m; + m.label = "OnPair" + std::to_string(bits); + // Realistic bit-packed accounting: codes packed at the true code width for the + // trained dictionary (not a fixed u16), dictionary offsets bit-packed, and the + // shared per-row length array (in place of the OnPair code-offset array). + size_t dict_bytes = col.dict.logical_bytes(); + size_t code_bits = IndexBits(col.dict.num_tokens()); + size_t codes = BitPackedBytes(col.codes.size(), code_bits); + size_t dict_offsets = + BitPackedBytes(col.dict.offsets.size(), std::max(1, BitWidth(dict_bytes))); + m.compressed_bytes = dict_bytes + dict_offsets + codes + c.len_array_bytes(); + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + op::Column tmp = op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), n, cfg); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(tmp.codes.size()) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Decode the *packed* code stream (unpack `code_bits` per code + gather), so + // decode pays the real bit-unpacking cost that the packed ratio implies. + size_t cap = op::DecodedLen(col) + op::kDecodePadding; + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), code_bits); + { + std::vector out(cap, 0); + size_t w = op::DecompressPacked(col.dict, packed.data(), col.codes.size(), code_bits, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "OnPair%u packed roundtrip mismatch on %s (w=%zu raw=%zu)\n", bits, + c.name.c_str(), w, c.raw_bytes()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap, 0); + auto t0 = Clock::now(); + size_t w = op::DecompressPacked(col.dict, packed.data(), col.codes.size(), code_bits, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// Bit-packed size of an OnPair column (dict + bit-packed dict offsets + codes at +// the true code width + the shared per-row length array). +size_t OnPairSize(const op::Column& col, const Corpus& c) { + size_t db = col.dict.logical_bytes(); + return db + BitPackedBytes(col.dict.offsets.size(), std::max(1, BitWidth(db))) + + BitPackedBytes(col.codes.size(), IndexBits(col.dict.num_tokens())) + c.len_array_bytes(); +} + +// OnPair with the dictionary bit-width chosen per column: try 9..16 and keep the +// width that minimizes bit-packed size, then report that width's ratio/decode. +// This exhaustive full-column sweep is the reliable way to pick the width. A +// cheap "train on a sub-sample and project to full size" picker does NOT +// reproduce it: training is not scale-invariant (the dynamic-threshold controller +// paces against the input size, so a sub-sample yields a differently *shaped* +// dictionary, not a smaller one), and enlarging the sample doesn't fix it - a +// token-gain curve fitted on a sample inherits the same skew. A cheap picker +// therefore needs a verify-against-the-ceiling fail-safe (train at the chosen +// budget and the ceiling, keep whichever stores less), not blind trust. +Measured RunOnPairAuto(const Corpus& c, double threshold) { + size_t n = c.n_rows(); + uint8_t best_bits = 9; + size_t best_sz = SIZE_MAX; + for (uint8_t b = 9; b <= 16; ++b) { + op::Config cfg{b, threshold, 42}; + op::Column col = op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), n, cfg); + size_t sz = OnPairSize(col, c); + if (sz < best_sz) { best_sz = sz; best_bits = b; } + } + + op::Config cfg{best_bits, threshold, 42}; + op::Column col = op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), n, cfg); + Measured m; + // Report the *stored* code width = ceil(log2(tokens trained)), which is what + // determines size. It can be < the budget when training saturates first. + size_t stored_bits = IndexBits(col.dict.num_tokens()); + m.label = "OnPair-auto(" + std::to_string(stored_bits) + "b)"; + m.compressed_bytes = OnPairSize(col, c); + + // Encode throughput at the chosen width (a real encoder adds only a cheap + // one-pass width estimate, not a full re-search, so this is representative). + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + op::Column tmp = op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), n, cfg); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(tmp.codes.size()) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Packed decode at the chosen stored width. + size_t cap = op::DecodedLen(col) + op::kDecodePadding; + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), stored_bits); + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap, 0); + auto t0 = Clock::now(); + size_t w = op::DecompressPacked(col.dict, packed.data(), col.codes.size(), stored_bits, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// dedup-then-OnPair +// +// The layout a real columnar format uses for repetitive columns: encode the +// column as bit-packed references into the set of distinct values, and run +// OnPair over only those distinct values. Removes whole-value repetition (which +// OnPair's <=16-byte substring dictionary can't exploit) before compressing the +// residual substring redundancy. Matches the OnPair README's guidance for +// low-cardinality columns. + +inline size_t CeilLog2(size_t x) { + if (x <= 1) return 1; // need >=1 bit even for a 2-value dictionary + return 64 - static_cast(__builtin_clzll(x - 1)); +} + +// Fast allocation-free byte-range hash, consuming 8 bytes per step (with a +// tail) and a final avalanche - far cheaper than a byte-at-a-time FNV for the +// short strings that dominate low-cardinality columns. +inline uint64_t HashBytes(const uint8_t* p, size_t len) { + uint64_t h = 0x9E3779B97F4A7C15ull ^ (static_cast(len) * 0xff51afd7ed558ccdull); + size_t i = 0; + for (; i + 8 <= len; i += 8) { + uint64_t w; + std::memcpy(&w, p + i, 8); + h = (h ^ w) * 0x100000001b3ull; + } + if (i < len) { + uint64_t w = 0; + std::memcpy(&w, p + i, len - i); + h = (h ^ w) * 0x100000001b3ull; + } + h ^= h >> 29; + h *= 0xbf58476d1ce4e5b9ull; + h ^= h >> 32; + return h; +} + +Measured RunOnPairDedup(const Corpus& c, uint8_t bits, double threshold) { + op::Config cfg; + cfg.max_dict_bits = bits; + cfg.threshold_fraction = threshold; + cfg.seed = 42; + size_t n = c.n_rows(); + + // Build the distinct-value set in first-seen order + per-row references. + // Open-addressing (linear-probe) table keyed on the row bytes, assigning ids + // in first-seen order - same distinct set/order as a std::unordered_map would + // give (so ratios are identical) but without per-key node allocation or the + // std::hash + pointer-chase overhead, which dominated encode. + auto build_dedup = [&](std::vector* d_bytes, std::vector* d_offsets, + std::vector* refs) { + size_t cap = 1; + while (cap < n * 2) cap <<= 1; // power-of-two, <=50% load + const uint32_t kEmpty = 0xFFFFFFFFu; + std::vector table(cap, kEmpty); // slot -> distinct id + uint64_t mask = cap - 1; + d_offsets->push_back(0); + refs->resize(n); + uint32_t n_distinct = 0; + for (size_t i = 0; i < n; ++i) { + const uint8_t* row = c.bytes.data() + c.offsets[i]; + size_t len = c.offsets[i + 1] - c.offsets[i]; + uint64_t slot = HashBytes(row, len) & mask; + uint32_t id; + for (;;) { + uint32_t cur = table[slot]; + if (cur == kEmpty) { // new distinct value + id = n_distinct++; + table[slot] = id; + d_bytes->insert(d_bytes->end(), row, row + len); + d_offsets->push_back(static_cast(d_bytes->size())); + break; + } + size_t off = (*d_offsets)[cur]; + size_t clen = (*d_offsets)[cur + 1] - off; + if (clen == len && std::memcmp(d_bytes->data() + off, row, len) == 0) { + id = cur; // seen before + break; + } + slot = (slot + 1) & mask; // linear probe + } + (*refs)[i] = id; + } + return static_cast(n_distinct); + }; + + std::vector d_bytes; + std::vector d_offsets; + std::vector refs; + size_t n_distinct = build_dedup(&d_bytes, &d_offsets, &refs); + + op::Column col = op::Compress(d_bytes.data(), d_bytes.size(), d_offsets.data(), n_distinct, cfg); + + Measured m; + m.label = "OnPair" + std::to_string(bits) + "-dedup"; + // Realistic bit-packed accounting, applied to the OnPair-encoded distinct set + // (dict + true-width codes + bit-packed dict offsets + a distinct-value length + // array) plus the bit-packed per-row reference (index) column. + size_t dict_bytes = col.dict.logical_bytes(); + size_t code_bits = IndexBits(col.dict.num_tokens()); + size_t codes = BitPackedBytes(col.codes.size(), code_bits); + size_t dict_offsets = + BitPackedBytes(col.dict.offsets.size(), std::max(1, BitWidth(dict_bytes))); + size_t dmax = 0; + for (size_t j = 0; j + 1 < d_offsets.size(); ++j) + dmax = std::max(dmax, d_offsets[j + 1] - d_offsets[j]); + size_t distinct_len_bytes = BitPackedBytes(n_distinct, std::max(1, BitWidth(dmax))); + size_t onpair_bytes = dict_bytes + dict_offsets + codes + distinct_len_bytes; + size_t refs_bytes = BitPackedBytes(n, IndexBits(n_distinct)); // index column + m.compressed_bytes = onpair_bytes + refs_bytes; + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + std::vector db; + std::vector doff; + std::vector rf; + auto t0 = Clock::now(); + size_t nd = build_dedup(&db, &doff, &rf); + op::Column tmp = op::Compress(db.data(), db.size(), doff.data(), nd, cfg); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(tmp.codes.size()) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Decode: materialize distinct values once (OnPair decode), then gather rows + // by reference. dbuf[d_offsets[id]..] holds distinct value `id` (decode + // reproduces the concatenated distinct bytes in id order). + // + // Gather uses a branchless fixed-16-byte copy when the value is <=16 bytes + // (the common case for the low-cardinality columns where dedup shines): one + // 128-bit store instead of a variable-length memcpy dispatch. Safe because the + // OnPair decode buffer is read-padded by kDecodePadding(16) and `out` carries + // 16 bytes of write padding; the cursor advances by the true length so the + // over-store is overwritten by the next row (or absorbed by the pad on the + // last). Values >16 bytes fall back to an exact memcpy. + size_t dlen = op::DecodedLen(col); + size_t cap = c.raw_bytes() + 16; + // Pack the distinct-set code stream and the per-row reference (index) column, + // so decode pays the real unpacking cost the packed sizes imply. + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed_codes = op::PackValues(cw.data(), cw.size(), code_bits); + size_t ref_bits = IndexBits(n_distinct); + std::vector packed_refs = op::PackValues(refs.data(), n, ref_bits); + + // Materialize the distinct values (unpack the OnPair code stream), then gather + // each row by unpacking its reference and copying the referenced value. <=16-byte + // values use one branchless 128-bit store (dbuf/out are 16-byte padded). + auto decode = [&](uint8_t* dbuf, uint8_t* out) -> size_t { + op::DecompressPacked(col.dict, packed_codes.data(), col.codes.size(), code_bits, dbuf); + size_t w = 0, bp = 0; + for (size_t i = 0; i < n; ++i) { + uint32_t id = op::GetBits(packed_refs.data(), bp, ref_bits); + bp += ref_bits; + size_t off = d_offsets[id]; + size_t len = d_offsets[id + 1] - off; + const uint8_t* src = dbuf + off; + if (len <= 16) std::memcpy(out + w, src, 16); + else std::memcpy(out + w, src, len); + w += len; + } + return w; + }; + + { + std::vector dbuf(dlen + op::kDecodePadding, 0), out(cap); + size_t w = decode(dbuf.data(), out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "OnPair%u-dedup packed roundtrip mismatch on %s\n", bits, c.name.c_str()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector dbuf(dlen + op::kDecodePadding, 0), out(cap); + auto t0 = Clock::now(); + size_t w = decode(dbuf.data(), out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// FSST+ / OnPair+ (common-prefix extraction, DICT mode) +// +// Y. L. Alexandre, "FSST+: Enhancing String Compression Through Common Prefix +// Extraction," MSc thesis, CWI, 2025. We evaluate the thesis's "DICT FSST+" +// path (the one it recommends for columnar integration, sec 5.3.5): the column +// is dictionary-encoded (distinct values + bit-packed row references), the +// distinct set is sorted, and prefix extraction is applied to that sorted +// dictionary. Sorting the dictionary is free of row-order concerns because the +// references carry the mapping - which is exactly why the thesis notes the +// within-block-sort limitation "does not apply" to DICT FSST+. +// +// The DP cleaving (prefix_plus::CleaveSorted, thesis sec 5.2.2) is shared. FSST+ +// cleaves the FSST-compressed distinct values (prefix/suffix are spans of the +// compressed stream; the escape-255 guard keeps a symbol whole). OnPair+ cleaves +// raw bytes and OnPair-compresses the {shared prefixes + per-value suffixes} as +// one column - a deviation forced by OnPair emitting codes, not a splittable +// byte stream. Both count a bit-packed row-reference (index) column, matching +// OnPair16-dedup, so ratios are directly comparable. + +// Shared distinct-value builder (first-seen order) + per-row references. Same +// open-addressing table as RunOnPairDedup's local build_dedup. +struct Dedup { + std::vector bytes; + std::vector offsets; // n_distinct + 1 + std::vector refs; // n_rows -> distinct id + size_t n_distinct = 0; +}; + +Dedup BuildDedup(const Corpus& c) { + size_t n = c.n_rows(); + Dedup d; + d.offsets.push_back(0); + d.refs.resize(n); + size_t cap = 1; + while (cap < n * 2) cap <<= 1; + const uint32_t kEmpty = 0xFFFFFFFFu; + std::vector table(cap, kEmpty); + uint64_t mask = cap - 1; + uint32_t n_distinct = 0; + for (size_t i = 0; i < n; ++i) { + const uint8_t* row = c.bytes.data() + c.offsets[i]; + size_t len = c.offsets[i + 1] - c.offsets[i]; + uint64_t slot = HashBytes(row, len) & mask; + uint32_t id; + for (;;) { + uint32_t cur = table[slot]; + if (cur == kEmpty) { + id = n_distinct++; + table[slot] = id; + d.bytes.insert(d.bytes.end(), row, row + len); + d.offsets.push_back(static_cast(d.bytes.size())); + break; + } + size_t off = d.offsets[cur]; + size_t clen = d.offsets[cur + 1] - off; + if (clen == len && std::memcmp(d.bytes.data() + off, row, len) == 0) { id = cur; break; } + slot = (slot + 1) & mask; + } + d.refs[i] = id; + } + d.n_distinct = n_distinct; + return d; +} + +namespace pp = parquet::prefix_plus; + +// ---- FSST+ ---------------------------------------------------------------- + +struct FsstPlusEnc { + std::vector comp; // FSST-compressed distinct values (spans by id) + std::vector comp_off; // distinct id -> byte offset into comp + std::vector comp_len; // distinct id -> compressed length + std::vector table; // fsst_export symbol table + size_t table_bytes = 0; + fsst_decoder_t dec{}; + std::vector order; // sorted rank -> distinct id (by compressed bytes) + std::vector sorted_pos; // distinct id -> sorted rank + pp::Cleaving cl; // over the sorted compressed spans + size_t nd = 0; +}; + +FsstPlusEnc EncodeFsstPlus(const Corpus& c, const Dedup& dd) { + FsstPlusEnc e; + size_t nd = dd.n_distinct; + e.nd = nd; + + std::vector lenIn(nd); + std::vector strIn(nd); + for (size_t i = 0; i < nd; ++i) { + lenIn[i] = dd.offsets[i + 1] - dd.offsets[i]; + strIn[i] = dd.bytes.data() + dd.offsets[i]; + } + fsst_encoder_t* enc = fsst_create(nd, lenIn.data(), strIn.data(), 0); + size_t out_cap = 7 * nd + 2 * dd.bytes.size() + 16; + e.comp.assign(out_cap, 0); + std::vector lenOut(nd); + std::vector strOut(nd); + size_t done = fsst_compress(enc, nd, lenIn.data(), strIn.data(), out_cap, e.comp.data(), + lenOut.data(), strOut.data()); + if (done != nd) { + std::fprintf(stderr, "FSST+ compressed %zu/%zu distinct on %s\n", done, nd, c.name.c_str()); + std::abort(); + } + e.comp_off.resize(nd); + e.comp_len.resize(nd); + for (size_t i = 0; i < nd; ++i) { + e.comp_off[i] = static_cast(strOut[i] - e.comp.data()); + e.comp_len[i] = static_cast(lenOut[i]); + } + unsigned char tbl[FSST_MAXHEADER]; + e.table_bytes = fsst_export(enc, tbl); + e.table.assign(tbl, tbl + e.table_bytes); + e.dec = fsst_decoder(enc); + fsst_destroy(enc); + + // Sort distinct ids by their compressed bytes so shared compressed prefixes + // are adjacent (FSST maps equal inputs to equal compressed forms). + e.order.resize(nd); + for (size_t i = 0; i < nd; ++i) e.order[i] = static_cast(i); + const uint8_t* base = e.comp.data(); + std::sort(e.order.begin(), e.order.end(), [&](uint32_t a, uint32_t b) { + size_t la = e.comp_len[a], lb = e.comp_len[b]; + int cmp = std::memcmp(base + e.comp_off[a], base + e.comp_off[b], std::min(la, lb)); + if (cmp != 0) return cmp < 0; + return la < lb; + }); + e.sorted_pos.resize(nd); + for (size_t k = 0; k < nd; ++k) e.sorted_pos[e.order[k]] = static_cast(k); + + std::vector sptr(nd); + std::vector slen(nd); + for (size_t k = 0; k < nd; ++k) { + sptr[k] = base + e.comp_off[e.order[k]]; + slen[k] = e.comp_len[e.order[k]]; + } + e.cl = pp::CleaveSorted(sptr.data(), slen.data(), nd, pp::kMaxPrefix, /*guard_escape255=*/true); + return e; +} + +// Exact FSST+ stored size (thesis sec 3.1 layout) + bit-packed row references. +size_t FsstPlusSize(const FsstPlusEnc& e, const Corpus& c) { + size_t nd = e.nd; + size_t num_blocks = (nd + pp::kBlockSize - 1) / pp::kBlockSize; + size_t bytes = 2 + 4 * num_blocks + 4; // num_blocks + block_start_offsets[] + data_end_offset + for (size_t bstart = 0; bstart < nd; bstart += pp::kBlockSize) { + size_t bn = std::min(pp::kBlockSize, nd - bstart); + bytes += 1 + 2 * bn; // num_strings + suffix_data_area_offsets[] + for (size_t k = bstart; k < bstart + bn; ++k) { + uint32_t p = e.cl.prefix_len[k]; + uint32_t clen = e.comp_len[e.order[k]]; + bytes += 1; // prefix_length + if (p > 0) bytes += 2; // jump_back_offset + bytes += clen - p; // compressed suffix + if (p > 0 && e.cl.chunk_first[k] == k) bytes += p; // shared prefix, stored once + } + } + bytes += e.table_bytes; + return bytes + BitPackedBytes(c.n_rows(), IndexBits(nd)); +} + +Measured RunFsstPlus(const Corpus& c) { + size_t n = c.n_rows(); + Dedup dd = BuildDedup(c); + FsstPlusEnc e = EncodeFsstPlus(c, dd); + Measured m; + m.label = "FSST+"; + m.compressed_bytes = FsstPlusSize(e, c); + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + Dedup d2 = BuildDedup(c); + FsstPlusEnc e2 = EncodeFsstPlus(c, d2); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(e2.nd) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Decode: reconstruct each sorted distinct value from its stored (prefix once + // + own suffix) compressed form and FSST-decode it, then gather rows by ref. + size_t distinct_total = dd.bytes.size(); + size_t cap = c.raw_bytes() + 16; + const uint8_t* base = e.comp.data(); + auto decode = [&](std::vector& dictbuf, std::vector& voff, + uint8_t* out) -> size_t { + voff.assign(e.nd + 1, 0); + std::vector tmp; + tmp.reserve(pp::kMaxPrefix + 256); + size_t w = 0; + for (size_t k = 0; k < e.nd; ++k) { + uint32_t p = e.cl.prefix_len[k]; + uint32_t rep = e.cl.chunk_first[k]; + uint32_t clen = e.comp_len[e.order[k]]; + const uint8_t* cbytes = base + e.comp_off[e.order[k]]; + tmp.clear(); + if (p > 0) { + const uint8_t* rbytes = base + e.comp_off[e.order[rep]]; + tmp.insert(tmp.end(), rbytes, rbytes + p); + } + tmp.insert(tmp.end(), cbytes + p, cbytes + clen); + size_t dl = fsst_decompress(&e.dec, tmp.size(), tmp.data(), dictbuf.size() - w, + dictbuf.data() + w); + w += dl; + voff[k + 1] = static_cast(w); + } + size_t o = 0; + for (size_t i = 0; i < n; ++i) { + uint32_t k = e.sorted_pos[dd.refs[i]]; + size_t off = voff[k], len = voff[k + 1] - off; + std::memcpy(out + o, dictbuf.data() + off, len); + o += len; + } + return o; + }; + + { + std::vector dictbuf(distinct_total + 32, 0), out(cap); + std::vector voff; + size_t w = decode(dictbuf, voff, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "FSST+ roundtrip mismatch on %s (w=%zu raw=%zu)\n", c.name.c_str(), w, + c.raw_bytes()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector dictbuf(distinct_total + 32, 0), out(cap); + std::vector voff; + auto t0 = Clock::now(); + size_t w = decode(dictbuf, voff, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// ---- OnPair+ -------------------------------------------------------------- + +struct OnPairPlusEnc { + op::Column col; // {shared prefixes + suffixes} as one column + std::vector piece_off; // num_pieces + 1 byte offsets (also decoded boundaries) + size_t num_pieces = 0; + std::vector order, sorted_pos; // distinct id <-> sorted rank (by raw bytes) + pp::Cleaving cl; // over raw sorted values + std::vector prefix_piece; // sorted rank of a chunk rep -> its prefix piece index + size_t n_prefix_pieces = 0; // suffix piece of sorted value k == n_prefix_pieces + k + size_t nd = 0; +}; + +OnPairPlusEnc EncodeOnPairPlus(const Corpus& c, const Dedup& dd, double threshold) { + (void)c; + OnPairPlusEnc e; + size_t nd = dd.n_distinct; + e.nd = nd; + + const uint8_t* base = dd.bytes.data(); + e.order.resize(nd); + for (size_t i = 0; i < nd; ++i) e.order[i] = static_cast(i); + std::sort(e.order.begin(), e.order.end(), [&](uint32_t a, uint32_t b) { + size_t la = dd.offsets[a + 1] - dd.offsets[a], lb = dd.offsets[b + 1] - dd.offsets[b]; + int cmp = std::memcmp(base + dd.offsets[a], base + dd.offsets[b], std::min(la, lb)); + if (cmp != 0) return cmp < 0; + return la < lb; + }); + e.sorted_pos.resize(nd); + for (size_t k = 0; k < nd; ++k) e.sorted_pos[e.order[k]] = static_cast(k); + + std::vector sptr(nd); + std::vector slen(nd); + for (size_t k = 0; k < nd; ++k) { + uint32_t id = e.order[k]; + sptr[k] = base + dd.offsets[id]; + slen[k] = dd.offsets[id + 1] - dd.offsets[id]; + } + e.cl = pp::CleaveSorted(sptr.data(), slen.data(), nd, pp::kMaxPrefix, /*guard_escape255=*/false); + + // Pieces: each chunk's shared prefix once, then every value's suffix. + std::vector pbytes; + std::vector poff; + poff.push_back(0); + e.prefix_piece.assign(nd, 0xFFFFFFFFu); + uint32_t pc = 0; + for (size_t k = 0; k < nd; ++k) { + if (e.cl.prefix_len[k] > 0 && e.cl.chunk_first[k] == k) { + e.prefix_piece[k] = pc++; + pbytes.insert(pbytes.end(), sptr[k], sptr[k] + e.cl.prefix_len[k]); + poff.push_back(static_cast(pbytes.size())); + } + } + e.n_prefix_pieces = pc; + for (size_t k = 0; k < nd; ++k) { + uint32_t p = e.cl.prefix_len[k]; + pbytes.insert(pbytes.end(), sptr[k] + p, sptr[k] + slen[k]); + poff.push_back(static_cast(pbytes.size())); + } + e.num_pieces = poff.size() - 1; + e.piece_off = std::move(poff); + + op::Config cfg{16, threshold, 42}; + e.col = op::Compress(pbytes.data(), pbytes.size(), e.piece_off.data(), e.num_pieces, cfg); + return e; +} + +// OnPair+ stored size: the shared OnPair model (dict + bit-packed offsets + +// codes for prefixes-once + suffixes), a bit-packed piece-boundary array (in +// place of FSST+'s compressed byte spans), the same per-value prefix_length / +// jump-back overhead and block headers as FSST+, and the row-reference column. +size_t OnPairPlusSize(const OnPairPlusEnc& e, const Corpus& c) { + const op::Column& col = e.col; + size_t dict_bytes = col.dict.logical_bytes(); + size_t code_bits = IndexBits(col.dict.num_tokens()); + size_t codes = BitPackedBytes(col.codes.size(), code_bits); + size_t dict_offsets = + BitPackedBytes(col.dict.offsets.size(), std::max(1, BitWidth(dict_bytes))); + size_t piece_bound = + BitPackedBytes(e.num_pieces, std::max(1, BitWidth(col.codes.size()))); + size_t nd = e.nd; + size_t num_blocks = (nd + pp::kBlockSize - 1) / pp::kBlockSize; + size_t structural = 2 + 4 * num_blocks + 4; + for (size_t bstart = 0; bstart < nd; bstart += pp::kBlockSize) { + size_t bn = std::min(pp::kBlockSize, nd - bstart); + structural += 1; // num_strings + for (size_t k = bstart; k < bstart + bn; ++k) + structural += 1 + (e.cl.prefix_len[k] > 0 ? 2 : 0); + } + size_t refs = BitPackedBytes(c.n_rows(), IndexBits(nd)); + return dict_bytes + dict_offsets + codes + piece_bound + structural + refs; +} + +Measured RunOnPairPlus(const Corpus& c, double threshold) { + size_t n = c.n_rows(); + Dedup dd = BuildDedup(c); + OnPairPlusEnc e = EncodeOnPairPlus(c, dd, threshold); + Measured m; + m.label = "OnPair+"; + m.compressed_bytes = OnPairPlusSize(e, c); + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + Dedup d2 = BuildDedup(c); + OnPairPlusEnc e2 = EncodeOnPairPlus(c, d2, threshold); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(e2.num_pieces) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + // Decode: OnPair-materialize all pieces (unpacking the code stream), then for + // each sorted value concatenate its prefix piece + suffix piece, and gather + // rows by reference. + size_t piece_total = op::DecodedLen(e.col); + size_t distinct_total = dd.bytes.size(); + size_t cap = c.raw_bytes() + 16; + size_t code_bits = IndexBits(e.col.dict.num_tokens()); + std::vector cw(e.col.codes.begin(), e.col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), code_bits); + + auto decode = [&](std::vector& piecebuf, std::vector& dictbuf, + std::vector& voff, uint8_t* out) -> size_t { + op::DecompressPacked(e.col.dict, packed.data(), e.col.codes.size(), code_bits, piecebuf.data()); + voff.assign(e.nd + 1, 0); + size_t w = 0; + for (size_t k = 0; k < e.nd; ++k) { + uint32_t p = e.cl.prefix_len[k]; + if (p > 0) { + uint32_t pi = e.prefix_piece[e.cl.chunk_first[k]]; + std::memcpy(dictbuf.data() + w, piecebuf.data() + e.piece_off[pi], p); + w += p; + } + uint32_t sfx = static_cast(e.n_prefix_pieces) + static_cast(k); + size_t soff = e.piece_off[sfx], slen = e.piece_off[sfx + 1] - soff; + std::memcpy(dictbuf.data() + w, piecebuf.data() + soff, slen); + w += slen; + voff[k + 1] = static_cast(w); + } + size_t o = 0; + for (size_t i = 0; i < n; ++i) { + uint32_t k = e.sorted_pos[dd.refs[i]]; + size_t off = voff[k], len = voff[k + 1] - off; + std::memcpy(out + o, dictbuf.data() + off, len); + o += len; + } + return o; + }; + + { + std::vector piecebuf(piece_total + op::kDecodePadding, 0); + std::vector dictbuf(distinct_total + 32, 0), out(cap); + std::vector voff; + size_t w = decode(piecebuf, dictbuf, voff, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "OnPair+ roundtrip mismatch on %s (w=%zu raw=%zu)\n", c.name.c_str(), w, + c.raw_bytes()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector piecebuf(piece_total + op::kDecodePadding, 0); + std::vector dictbuf(distinct_total + 32, 0), out(cap); + std::vector voff; + auto t0 = Clock::now(); + size_t w = decode(piecebuf, dictbuf, voff, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +} // namespace + +int main(int argc, char** argv) { + std::string dir = CorpusDir(argc, argv); + std::vector files = CorpusFiles(dir); + if (files.empty()) { + std::fprintf(stderr, "no .txt corpora in %s\n", dir.c_str()); + return 1; + } + + std::printf("%-26s %10s %10s %7s %9s %9s\n", "corpus", "rows", "raw MiB", "ratio", + "enc MiB/s", "dec MiB/s"); + std::printf("%s\n", std::string(90, '-').c_str()); + + for (const auto& path : files) { + Corpus c = ReadCorpus(path); + double threshold = ThresholdFor(c.name); + + Measured fsst = RunFsst(c); + Measured zstd1 = RunZstd(c, 1); + Measured lz4 = RunLz4(c); + Measured op16 = RunOnPair(c, 16, threshold); + Measured opauto = RunOnPairAuto(c, threshold); + Measured op16d = RunOnPairDedup(c, 16, threshold); + Measured fsstp = RunFsstPlus(c); + Measured oppl = RunOnPairPlus(c, threshold); + + std::printf("%-26s %10zu %10.2f\n", c.name.c_str(), c.n_rows(), Mib(c.raw_bytes())); + for (const Measured* m : {&fsst, &zstd1, &lz4, &op16, &opauto, &op16d, &fsstp, &oppl}) { + double ratio = static_cast(c.raw_bytes()) / static_cast(m->compressed_bytes); + std::printf(" %-24s %10s %10.2f %7.3fx %9.1f %9.1f\n", m->label.c_str(), "", + Mib(m->compressed_bytes), ratio, m->encode_mibs, m->decode_mibs); + } + double r_fsst = static_cast(c.raw_bytes()) / fsst.compressed_bytes; + double r_zstd = static_cast(c.raw_bytes()) / zstd1.compressed_bytes; + double r_op16 = static_cast(c.raw_bytes()) / op16.compressed_bytes; + std::printf(" -> OnPair16 vs FSST: ratio %+.1f%%, encode %+.1f%%, decode %+.1f%%\n", + (r_op16 / r_fsst - 1.0) * 100.0, (op16.encode_mibs / fsst.encode_mibs - 1.0) * 100.0, + (op16.decode_mibs / fsst.decode_mibs - 1.0) * 100.0); + std::printf(" -> OnPair16 vs zstd(1): ratio %+.1f%%, encode %+.1f%%, decode %+.1f%%\n", + (r_op16 / r_zstd - 1.0) * 100.0, + (op16.encode_mibs / zstd1.encode_mibs - 1.0) * 100.0, + (op16.decode_mibs / zstd1.decode_mibs - 1.0) * 100.0); + double r_op16d = static_cast(c.raw_bytes()) / op16d.compressed_bytes; + std::printf(" -> OnPair16-dedup vs zstd(1): ratio %+.1f%%, decode %+.1f%%\n", + (r_op16d / r_zstd - 1.0) * 100.0, + (op16d.decode_mibs / zstd1.decode_mibs - 1.0) * 100.0); + double r_fsstp = static_cast(c.raw_bytes()) / fsstp.compressed_bytes; + double r_oppl = static_cast(c.raw_bytes()) / oppl.compressed_bytes; + std::printf(" -> FSST+ vs FSST: ratio %+.1f%%; FSST+ vs zstd(1): ratio %+.1f%%\n", + (r_fsstp / r_fsst - 1.0) * 100.0, (r_fsstp / r_zstd - 1.0) * 100.0); + std::printf(" -> OnPair+ vs OnPair16-dedup: ratio %+.1f%%; OnPair+ vs zstd(1): ratio %+.1f%%\n\n", + (r_oppl / r_op16d - 1.0) * 100.0, (r_oppl / r_zstd - 1.0) * 100.0); + } + return 0; +} From d5f384de4b18094acdacd0549d3f9a50a4ba39f1 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 3 Aug 2026 06:02:20 +0000 Subject: [PATCH 20/24] Add a cascade benchmark for encoding plus generic-codec combinations Measures the native Parquet pages as libparquet writes them, each on its own and followed by zstd(1) or lz4, against FSST and OnPair with and without a generic codec on top. Also measures the dictionary-then-OnPair cascade, which dictionary-encodes the column and OnPairs only the distinct values. Native pages are charged only what the writer emits, since they carry their own lengths; the candidate codecs are charged a separate length array. Auto-budget selection runs in per-corpus setup, outside the timed region, as every codec's parameter choice does. --- cpp/src/parquet/onpair/cascade_benchmark.cc | 963 ++++++++++++++++++++ 1 file changed, 963 insertions(+) create mode 100644 cpp/src/parquet/onpair/cascade_benchmark.cc diff --git a/cpp/src/parquet/onpair/cascade_benchmark.cc b/cpp/src/parquet/onpair/cascade_benchmark.cc new file mode 100644 index 000000000000..6e088f976945 --- /dev/null +++ b/cpp/src/parquet/onpair/cascade_benchmark.cc @@ -0,0 +1,963 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Does it pay to cascade a generic codec (zstd/lz4) over a string encoding? +// +// The companion benchmark (fsst_onpair_benchmark.cc) measures FSST, OnPair, zstd +// and lz4 as alternatives. This one measures them as *compositions*, which is +// what a Parquet writer actually does: an encoding produces a page payload and a +// page compressor then runs over the whole payload. Two groups: +// +// A. FSST / OnPair followed by zstd(1) or lz4. Note what this costs: the +// generic layer is whole-page, so the per-row random access that is the +// reason to pick FSST or OnPair in the first place is gone -- you must +// inflate the page before touching one row. The question is whether the +// extra ratio pays for that. OnPair is also measured with a byte-aligned +// layout (u16 codes, u32 dictionary offsets) rather than bit-packed, +// because bit-packed streams are close to incompressible and testing only +// the packed layout would understate what a cascade can do. +// +// B. Parquet's own byte-array encodings -- PLAIN, DELTA_LENGTH_BYTE_ARRAY, +// DELTA_BYTE_ARRAY, RLE_DICTIONARY -- alone and under zstd(1)/lz4, driven +// through the real parquet::Encoder/Decoder API rather than reimplemented. +// PLAIN+ZSTD is what Parquet writes for string columns today, so this is +// the baseline a new encoding has to beat; the companion benchmark's +// zstd-over-concatenated-bytes column is a projection of a page, not one. +// +// Accounting matches the companion benchmark so ratios are comparable across the +// two binaries (see bench_common.h), with one deliberate difference: PLAIN and +// the DELTA_* family embed their own lengths, so they are NOT charged the +// separate bit-packed row-length array that FSST/zstd/lz4/OnPair are. Charging +// it would count row boundaries twice. Every number here is therefore the +// complete page payload needed to reconstruct the column. +// +// Decode is timed as "reconstruct the whole column into a contiguous buffer", +// including the generic decompression and the per-page decoder setup. The +// contiguous copy matters for the Parquet encodings: their Decode() hands back +// pointers into the decoder's own buffer, so timing that alone would report PLAIN +// as nearly free rather than as the memcpy it is. +// +// Build (from the Arrow repo root), one line -- needs libparquet, unlike the +// companion benchmark. libarrow is linked by path because the build directory +// carries two sonames and letting -larrow choose warns about the conflict: +// g++ -std=c++17 -O3 -march=native -Icpp/src -Icpp/build-bench/src +// -Icpp/thirdparty/fsst cpp/thirdparty/fsst/libfsst.cpp +// cpp/thirdparty/fsst/fsst_avx512.cpp cpp/src/parquet/onpair/onpair.cc +// cpp/src/parquet/onpair/cascade_benchmark.cc +// cpp/build-bench/release/libparquet.so cpp/build-bench/release/libarrow.so.2300 +// /usr/lib64/libzstd.so.1 /usr/lib64/liblz4.so.1 +// -Wl,-rpath,$PWD/cpp/build-bench/release -o /tmp/cascade_bench +// +// Run: taskset -c 0 /tmp/cascade_bench (run twice, use the 2nd) + +#include +#include +#include +#include +#include +#include +#include + +#include "fsst.h" +#include "parquet/encoding.h" +#include "parquet/onpair/bench_common.h" +#include "parquet/onpair/onpair.h" +#include "parquet/schema.h" +#include "parquet/types.h" + +namespace op = parquet::onpair; + +namespace { + +using namespace bench; // NOLINT(build/namespaces) + +// The generic (page-level) codec layered over an encoding's payload. + +enum class Generic { kNone, kZstd1, kLz4 }; + +const char* GenericSuffix(Generic g) { + switch (g) { + case Generic::kNone: + return ""; + case Generic::kZstd1: + return "+zstd(1)"; + default: + return "+lz4"; + } +} + +std::vector GenericCompress(const uint8_t* src, size_t n, Generic g) { + if (g == Generic::kNone) return std::vector(src, src + n); + std::vector out; + if (g == Generic::kZstd1) { + out.resize(ZSTD_compressBound(n)); + size_t c = ZSTD_compress(out.data(), out.size(), src, n, 1); + if (ZSTD_isError(c)) { + std::fprintf(stderr, "zstd compress error\n"); + std::abort(); + } + out.resize(c); + } else { + out.resize(static_cast(LZ4_compressBound(static_cast(n)))); + int c = LZ4_compress_default(reinterpret_cast(src), + reinterpret_cast(out.data()), + static_cast(n), static_cast(out.size())); + if (c <= 0) { + std::fprintf(stderr, "lz4 compress error\n"); + std::abort(); + } + out.resize(static_cast(c)); + } + return out; +} + +// `raw_size` is the payload's uncompressed length, which a real page header +// carries, so knowing it here is not cheating. +void GenericDecompress(const uint8_t* src, size_t csize, uint8_t* dst, size_t raw_size, + Generic g) { + if (g == Generic::kNone) { + std::memcpy(dst, src, csize); + return; + } + if (g == Generic::kZstd1) { + size_t w = ZSTD_decompress(dst, raw_size, src, csize); + if (ZSTD_isError(w) || w != raw_size) { + std::fprintf(stderr, "zstd decompress error\n"); + std::abort(); + } + return; + } + int w = LZ4_decompress_safe(reinterpret_cast(src), + reinterpret_cast(dst), static_cast(csize), + static_cast(raw_size)); + if (w != static_cast(raw_size)) { + std::fprintf(stderr, "lz4 decompress error\n"); + std::abort(); + } +} + +// A codec under test: `build` produces the encoded page payload from the corpus +// (the encode side, training included), `decode` reconstructs the concatenated +// column bytes from a payload buffer and returns the byte count written. +using BuildFn = std::function()>; +using DecodeFn = std::function; + +// Extra slack after the payload copy: OnPair's packed-code reader over-reads up +// to 4 bytes past the last code, and its dictionary decode over-reads one token. +constexpr size_t kPad = 64; + +// `extra_bytes` is charged on top of the compressed payload without being part +// of it. Only used to reproduce the companion benchmark's zstd/lz4 accounting, +// which charges an uncompressed row-length array alongside a frame that holds +// only the value bytes. +Measured RunCodec(const Corpus& c, const std::string& label, Generic g, const BuildFn& build, + const DecodeFn& decode, size_t extra_bytes = 0) { + Measured m; + m.label = label; + + std::vector payload = build(); + std::vector comp = GenericCompress(payload.data(), payload.size(), g); + m.compressed_bytes = comp.size() + extra_bytes; + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + std::vector p = build(); + std::vector cc = GenericCompress(p.data(), p.size(), g); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(cc.size()) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + const size_t out_cap = c.raw_bytes() + op::kDecodePadding + kPad; + { + std::vector scratch(payload.size() + kPad, 0); + std::vector out(out_cap, 0); + GenericDecompress(comp.data(), comp.size(), scratch.data(), payload.size(), g); + size_t w = decode(scratch.data(), payload.size(), out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "%s roundtrip mismatch on %s (w=%zu raw=%zu)\n", m.label.c_str(), + c.name.c_str(), w, c.raw_bytes()); + std::abort(); + } + } + + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector scratch(payload.size() + kPad, 0); + std::vector out(out_cap, 0); + auto t0 = Clock::now(); + GenericDecompress(comp.data(), comp.size(), scratch.data(), payload.size(), g); + size_t w = decode(scratch.data(), payload.size(), out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + +// Little-endian fixed-width field helpers for the payload headers. The headers +// are a handful of bytes per page and are charged, so no codec gets a free ride +// on self-description. + +void PutU32(std::vector* v, uint32_t x) { + uint8_t b[4]; + std::memcpy(b, &x, 4); + v->insert(v->end(), b, b + 4); +} +uint32_t GetU32(const uint8_t* p) { + uint32_t x; + std::memcpy(&x, p, 4); + return x; +} + +// Append the bit-packed per-row length array that every value-preserving codec +// is charged for (see Corpus::len_array_bytes). +void AppendLengths(const Corpus& c, std::vector* v) { + size_t bits = std::max(1, BitWidth(c.max_row_len())); + std::vector lens(c.n_rows()); + for (size_t i = 0; i < c.n_rows(); ++i) lens[i] = c.offsets[i + 1] - c.offsets[i]; + std::vector packed = op::PackValues(lens.data(), lens.size(), bits); + packed.resize(BitPackedBytes(c.n_rows(), bits)); + v->insert(v->end(), packed.begin(), packed.end()); +} + +// Byte-aligned analog of the above: lengths at the smallest whole-byte width. +void AppendLengthsByteAligned(const Corpus& c, std::vector* v) { + size_t w = std::max(1, (BitWidth(c.max_row_len()) + 7) / 8); + for (size_t i = 0; i < c.n_rows(); ++i) { + uint32_t len = c.offsets[i + 1] - c.offsets[i]; + for (size_t b = 0; b < w; ++b) v->push_back(static_cast(len >> (8 * b))); + } +} + +// Group A codec 1: FSST +// +// Payload: [u32 table_bytes][u32 stream_bytes][symbol table][code stream][lengths] + +std::vector BuildFsstPayload(const Corpus& c) { + size_t n = c.n_rows(); + std::vector lenIn(n); + std::vector strIn(n); + for (size_t i = 0; i < n; ++i) { + lenIn[i] = c.offsets[i + 1] - c.offsets[i]; + strIn[i] = c.bytes.data() + c.offsets[i]; + } + fsst_encoder_t* enc = fsst_create(n, lenIn.data(), strIn.data(), 0); + + size_t out_cap = 7 * n + 2 * c.raw_bytes() + 16; // fsst.h bound: 7 + 2*len per string + std::vector stream(out_cap); + std::vector lenOut(n); + std::vector strOut(n); + size_t done = fsst_compress(enc, n, lenIn.data(), strIn.data(), out_cap, stream.data(), + lenOut.data(), strOut.data()); + if (done != n) { + std::fprintf(stderr, "FSST: only compressed %zu/%zu rows\n", done, n); + std::abort(); + } + size_t total = 0; + for (size_t i = 0; i < n; ++i) total += lenOut[i]; + + unsigned char table[FSST_MAXHEADER]; + size_t table_bytes = fsst_export(enc, table); + fsst_destroy(enc); + + std::vector v; + v.reserve(8 + table_bytes + total + c.len_array_bytes()); + PutU32(&v, static_cast(table_bytes)); + PutU32(&v, static_cast(total)); + v.insert(v.end(), table, table + table_bytes); + v.insert(v.end(), stream.begin(), stream.begin() + total); + AppendLengths(c, &v); + return v; +} + +size_t DecodeFsstPayload(const uint8_t* p, size_t /*size*/, uint8_t* out, size_t out_cap) { + size_t table_bytes = GetU32(p); + size_t stream_bytes = GetU32(p + 4); + fsst_decoder_t dec; + fsst_import(&dec, p + 8); + return fsst_decompress(&dec, stream_bytes, const_cast(p + 8 + table_bytes), + out_cap, out); +} + +// Group A codec 2: OnPair, bit-packed (the layout Tables 1-3 report) +// +// Payload: [u32 tokens][u32 dict_bytes][u32 codes][u8 code_bits][u8 off_bits][pad*2] +// [dictionary blob][packed dict offsets][packed codes][lengths] + +// Choose the dictionary budget the way the companion benchmark's OnPair-auto +// does: train at every width in 9..16 and keep the one that stores least. +size_t OnPairSize(const op::Column& col, const Corpus& c) { + size_t db = col.dict.logical_bytes(); + return db + BitPackedBytes(col.dict.offsets.size(), std::max(1, BitWidth(db))) + + BitPackedBytes(col.codes.size(), IndexBits(col.dict.num_tokens())) + + c.len_array_bytes(); +} + +uint8_t PickOnPairBits(const Corpus& c, double threshold) { + uint8_t best_bits = 9; + size_t best_sz = SIZE_MAX; + for (uint8_t b = 9; b <= 16; ++b) { + op::Config cfg{b, threshold, 42}; + op::Column col = + op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), c.n_rows(), cfg); + size_t sz = OnPairSize(col, c); + if (sz < best_sz) { + best_sz = sz; + best_bits = b; + } + } + return best_bits; +} + +constexpr size_t kOnPairHeader = 16; + +std::vector BuildOnPairPayload(const Corpus& c, const op::Config& cfg, + bool byte_aligned) { + op::Column col = + op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), c.n_rows(), cfg); + size_t dict_bytes = col.dict.logical_bytes(); + size_t code_bits = byte_aligned ? 16 : IndexBits(col.dict.num_tokens()); + size_t off_bits = byte_aligned ? 32 : std::max(1, BitWidth(dict_bytes)); + + std::vector v; + v.reserve(kOnPairHeader + dict_bytes + col.codes.size() * 2 + c.len_array_bytes()); + PutU32(&v, static_cast(col.dict.num_tokens())); + PutU32(&v, static_cast(dict_bytes)); + PutU32(&v, static_cast(col.codes.size())); + v.push_back(static_cast(code_bits)); + v.push_back(static_cast(off_bits)); + v.push_back(0); + v.push_back(0); + + v.insert(v.end(), col.dict.bytes.begin(), col.dict.bytes.begin() + dict_bytes); + + std::vector offs(col.dict.offsets.begin(), col.dict.offsets.end()); + std::vector packed_offs = op::PackValues(offs.data(), offs.size(), off_bits); + packed_offs.resize(BitPackedBytes(offs.size(), off_bits)); + v.insert(v.end(), packed_offs.begin(), packed_offs.end()); + + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed_codes = op::PackValues(cw.data(), cw.size(), code_bits); + packed_codes.resize(BitPackedBytes(cw.size(), code_bits)); + v.insert(v.end(), packed_codes.begin(), packed_codes.end()); + + if (byte_aligned) { + AppendLengthsByteAligned(c, &v); + } else { + AppendLengths(c, &v); + } + return v; +} + +// Rebuilds the dictionary from the payload (a reader must, since the decoder +// needs read-padded token bytes and materialized offsets) and decodes the packed +// code stream in place. +size_t DecodeOnPairPayload(const uint8_t* p, size_t /*size*/, uint8_t* out) { + size_t num_tokens = GetU32(p); + size_t dict_bytes = GetU32(p + 4); + size_t num_codes = GetU32(p + 8); + size_t code_bits = p[12]; + size_t off_bits = p[13]; + + const uint8_t* dict_blob = p + kOnPairHeader; + const uint8_t* packed_offs = dict_blob + dict_bytes; + size_t offs_bytes = BitPackedBytes(num_tokens + 1, off_bits); + const uint8_t* packed_codes = packed_offs + offs_bytes; + + op::CompactDictionary dict; + dict.bytes.resize(dict_bytes + op::kDecodePadding, 0); + std::memcpy(dict.bytes.data(), dict_blob, dict_bytes); + dict.offsets.resize(num_tokens + 1); + for (size_t t = 0; t <= num_tokens; ++t) { + dict.offsets[t] = op::GetBits(packed_offs, t * off_bits, off_bits); + } + dict.RecomputeMaxTokenLen(); + + return op::DecompressPacked(dict, packed_codes, num_codes, code_bits, out); +} + +// Group A codec 3: dictionary-encode, then OnPair the distinct values. +// +// The layout a columnar format uses for repetitive columns: bit-packed references +// into the distinct set, with OnPair run over only the distinct values. Measured +// here as well as in the companion benchmark so that a decode number for it can +// sit in the same table as the Parquet-native pages, which are 4.6-7.5% slower in +// this binary. +// +// Payload: [u32 distinct][u32 rows][u32 distinct_bytes][u8 ref_bits][u8 dlen_bits] +// [pad*2][OnPair payload of the distinct set][packed distinct lengths] +// [packed row references] +// +// Note what is NOT charged: the per-row length array. Row lengths are recovered +// from a row's reference plus the distinct-value lengths, so charging both would +// count boundaries twice. This matches the companion benchmark's accounting. + +// Allocation-free byte-range hash, 8 bytes per step plus a tail and a final +// avalanche -- cheaper than byte-at-a-time for the short values that dominate +// low-cardinality columns. Same function the companion benchmark uses, so both +// binaries build the identical distinct set in the identical order. +inline uint64_t HashBytes(const uint8_t* p, size_t len) { + uint64_t h = 0x9E3779B97F4A7C15ull ^ (static_cast(len) * 0xff51afd7ed558ccdull); + size_t i = 0; + for (; i + 8 <= len; i += 8) { + uint64_t w; + std::memcpy(&w, p + i, 8); + h = (h ^ w) * 0x100000001b3ull; + } + if (i < len) { + uint64_t w = 0; + std::memcpy(&w, p + i, len - i); + h = (h ^ w) * 0x100000001b3ull; + } + h ^= h >> 29; + h *= 0xbf58476d1ce4e5b9ull; + h ^= h >> 32; + return h; +} + +struct DedupSet { + std::vector bytes; // distinct values, concatenated in first-seen order + std::vector offsets; // n_distinct + 1 + std::vector refs; // one per row + size_t n_distinct = 0; +}; + +// Open-addressing (linear-probe) table keyed on the row bytes, assigning ids in +// first-seen order -- the same distinct set and order an unordered_map would +// give, so ratios match, without per-key allocation. +DedupSet BuildDedupSet(const Corpus& c) { + size_t n = c.n_rows(); + DedupSet d; + size_t cap = 1; + while (cap < n * 2) cap <<= 1; // power of two, <=50% load + const uint32_t kEmpty = 0xFFFFFFFFu; + std::vector table(cap, kEmpty); + uint64_t mask = cap - 1; + d.offsets.push_back(0); + d.refs.resize(n); + uint32_t n_distinct = 0; + for (size_t i = 0; i < n; ++i) { + const uint8_t* row = c.bytes.data() + c.offsets[i]; + size_t len = c.offsets[i + 1] - c.offsets[i]; + uint64_t slot = HashBytes(row, len) & mask; + uint32_t id; + for (;;) { + uint32_t cur = table[slot]; + if (cur == kEmpty) { + id = n_distinct++; + table[slot] = id; + d.bytes.insert(d.bytes.end(), row, row + len); + d.offsets.push_back(static_cast(d.bytes.size())); + break; + } + size_t off = d.offsets[cur]; + size_t clen = d.offsets[cur + 1] - off; + if (clen == len && std::memcmp(d.bytes.data() + off, row, len) == 0) { + id = cur; + break; + } + slot = (slot + 1) & mask; // linear probe + } + d.refs[i] = id; + } + d.n_distinct = n_distinct; + return d; +} + +constexpr size_t kDictOnPairHeader = 16; + +std::vector BuildDictOnPairPayload(const Corpus& c, const op::Config& cfg) { + DedupSet d = BuildDedupSet(c); + op::Column col = op::Compress(d.bytes.data(), d.bytes.size(), d.offsets.data(), + d.n_distinct, cfg); + size_t dict_bytes = col.dict.logical_bytes(); + size_t code_bits = IndexBits(col.dict.num_tokens()); + size_t off_bits = std::max(1, BitWidth(dict_bytes)); + + size_t dmax = 0; + for (size_t j = 0; j + 1 < d.offsets.size(); ++j) + dmax = std::max(dmax, d.offsets[j + 1] - d.offsets[j]); + size_t dlen_bits = std::max(1, BitWidth(dmax)); + size_t ref_bits = IndexBits(d.n_distinct); + + std::vector v; + PutU32(&v, static_cast(d.n_distinct)); + PutU32(&v, static_cast(c.n_rows())); + PutU32(&v, static_cast(d.bytes.size())); + v.push_back(static_cast(ref_bits)); + v.push_back(static_cast(dlen_bits)); + v.push_back(0); + v.push_back(0); + + PutU32(&v, static_cast(col.dict.num_tokens())); + PutU32(&v, static_cast(dict_bytes)); + PutU32(&v, static_cast(col.codes.size())); + v.push_back(static_cast(code_bits)); + v.push_back(static_cast(off_bits)); + v.push_back(0); + v.push_back(0); + + v.insert(v.end(), col.dict.bytes.begin(), col.dict.bytes.begin() + dict_bytes); + + std::vector offs(col.dict.offsets.begin(), col.dict.offsets.end()); + std::vector packed_offs = op::PackValues(offs.data(), offs.size(), off_bits); + packed_offs.resize(BitPackedBytes(offs.size(), off_bits)); + v.insert(v.end(), packed_offs.begin(), packed_offs.end()); + + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed_codes = op::PackValues(cw.data(), cw.size(), code_bits); + packed_codes.resize(BitPackedBytes(cw.size(), code_bits)); + v.insert(v.end(), packed_codes.begin(), packed_codes.end()); + + std::vector dlens(d.n_distinct); + for (size_t j = 0; j < d.n_distinct; ++j) dlens[j] = d.offsets[j + 1] - d.offsets[j]; + std::vector packed_dlens = op::PackValues(dlens.data(), dlens.size(), dlen_bits); + packed_dlens.resize(BitPackedBytes(dlens.size(), dlen_bits)); + v.insert(v.end(), packed_dlens.begin(), packed_dlens.end()); + + std::vector packed_refs = op::PackValues(d.refs.data(), d.refs.size(), ref_bits); + packed_refs.resize(BitPackedBytes(d.refs.size(), ref_bits)); + v.insert(v.end(), packed_refs.begin(), packed_refs.end()); + return v; +} + +// The dedup cascade's own budget sweep. PickOnPairBits cannot be reused: it scores +// OnPairSize over the whole column, whereas this payload trains OnPair over the +// *distinct set* and then adds packed references and distinct-value lengths, so +// the width that stores the column least is not the width that stores this least. +// +// It scores the assembled payload rather than a size formula on purpose -- the +// payload has five sections, and a parallel formula would drift from +// BuildDictOnPairPayload the first time either one changed. The build is wasted +// work, but this runs in per-corpus setup, not in the timed region. +uint8_t PickDictOnPairBits(const Corpus& c, double threshold) { + uint8_t best_bits = 9; + size_t best_sz = SIZE_MAX; + for (uint8_t b = 9; b <= 16; ++b) { + size_t sz = BuildDictOnPairPayload(c, op::Config{b, threshold, 42}).size(); + if (sz < best_sz) { + best_sz = sz; + best_bits = b; + } + } + return best_bits; +} + +// Materialize the distinct values (one OnPair decode), rebuild their offsets from +// the packed length array, then gather rows by unpacking each reference. Values of +// <=16 bytes take one branchless 128-bit store; `scratch` and `out` are padded. +// +// Unlike the companion benchmark, the offset rebuild is inside the timed region: +// this payload is self-describing, so a reader really does pay it. The difference +// is a prefix sum over the distinct set, which is negligible where dedup wins and +// only matters on all-distinct columns, where dedup loses regardless. +size_t DecodeDictOnPairPayload(const uint8_t* p, uint8_t* out, + std::vector* scratch) { + size_t n_distinct = GetU32(p); + size_t n_rows = GetU32(p + 4); + size_t distinct_bytes = GetU32(p + 8); + size_t ref_bits = p[12]; + size_t dlen_bits = p[13]; + + const uint8_t* q = p + kDictOnPairHeader; + size_t num_tokens = GetU32(q); + size_t dict_bytes = GetU32(q + 4); + size_t num_codes = GetU32(q + 8); + size_t code_bits = q[12]; + size_t off_bits = q[13]; + + const uint8_t* dict_blob = q + kOnPairHeader; + const uint8_t* packed_offs = dict_blob + dict_bytes; + const uint8_t* packed_codes = packed_offs + BitPackedBytes(num_tokens + 1, off_bits); + const uint8_t* packed_dlens = packed_codes + BitPackedBytes(num_codes, code_bits); + const uint8_t* packed_refs = packed_dlens + BitPackedBytes(n_distinct, dlen_bits); + + op::CompactDictionary dict; + dict.bytes.resize(dict_bytes + op::kDecodePadding, 0); + std::memcpy(dict.bytes.data(), dict_blob, dict_bytes); + dict.offsets.resize(num_tokens + 1); + for (size_t t = 0; t <= num_tokens; ++t) { + dict.offsets[t] = op::GetBits(packed_offs, t * off_bits, off_bits); + } + dict.RecomputeMaxTokenLen(); + + if (scratch->size() < distinct_bytes + op::kDecodePadding) { + scratch->assign(distinct_bytes + op::kDecodePadding, 0); + } + op::DecompressPacked(dict, packed_codes, num_codes, code_bits, scratch->data()); + + std::vector doff(n_distinct + 1); + doff[0] = 0; + for (size_t j = 0; j < n_distinct; ++j) { + doff[j + 1] = doff[j] + op::GetBits(packed_dlens, j * dlen_bits, dlen_bits); + } + + const uint8_t* dbuf = scratch->data(); + size_t w = 0, bp = 0; + for (size_t i = 0; i < n_rows; ++i) { + uint32_t id = op::GetBits(packed_refs, bp, ref_bits); + bp += ref_bits; + size_t off = doff[id]; + size_t len = doff[id + 1] - off; + if (len <= 16) { + std::memcpy(out + w, dbuf + off, 16); + } else { + std::memcpy(out + w, dbuf + off, len); + } + w += len; + } + return w; +} + +// Group B: Parquet's own byte-array encodings, through the real encoder/decoder. + +std::shared_ptr ByteArrayDescr() { + auto node = parquet::schema::PrimitiveNode::Make("ba", parquet::Repetition::REQUIRED, + parquet::Type::BYTE_ARRAY); + return std::make_shared(node, /*max_definition_level=*/0, + /*max_repetition_level=*/0); +} + +std::vector CorpusValues(const Corpus& c) { + std::vector vals(c.n_rows()); + for (size_t i = 0; i < c.n_rows(); ++i) { + vals[i] = parquet::ByteArray(c.offsets[i + 1] - c.offsets[i], c.bytes.data() + c.offsets[i]); + } + return vals; +} + +// Copy the decoded ByteArrays into one contiguous buffer. PLAIN and +// DELTA_LENGTH_BYTE_ARRAY hand back pointers into the page, so without this a +// "decode" would be pointer arithmetic and the comparison meaningless. +size_t Gather(const std::vector& vals, uint8_t* out) { + size_t w = 0; + for (const parquet::ByteArray& v : vals) { + std::memcpy(out + w, v.ptr, v.len); + w += v.len; + } + return w; +} + +// PLAIN / DELTA_LENGTH_BYTE_ARRAY / DELTA_BYTE_ARRAY: a single self-describing +// buffer, so nothing is added to it -- these encodings carry their own lengths. +std::vector BuildPqPayload(const Corpus& c, + const std::vector& vals, + parquet::Encoding::type e) { + auto enc = parquet::MakeTypedEncoder(e); + enc->Put(vals.data(), static_cast(c.n_rows())); + auto buf = enc->FlushValues(); + return std::vector(buf->data(), buf->data() + buf->size()); +} + +size_t DecodePqPayload(const Corpus& c, parquet::Encoding::type e, const uint8_t* p, + size_t size, uint8_t* out) { + int n = static_cast(c.n_rows()); + auto dec = parquet::MakeTypedDecoder(e); + dec->SetData(n, p, static_cast(size)); + std::vector vals(n); + int got = dec->Decode(vals.data(), n); + if (got != n) { + std::fprintf(stderr, "parquet decode short read (%d of %d)\n", got, n); + std::abort(); + } + return Gather(vals, out); +} + +// RLE_DICTIONARY is two streams -- a dictionary page and an index page -- so the +// payload concatenates them behind a header, and both are charged. +// +// Payload: [u32 dict_bytes][u32 idx_bytes][u32 num_entries][u32 dict page encoding] +// [dict page][indices] +// +// `dict_page_enc` is PLAIN for the RLE_DICTIONARY rows, which is the only thing a +// writer may emit: a BYTE_ARRAY dictionary page is PLAIN-encoded by spec. Passing +// anything else measures a *hypothetical* format change -- the fair opponent for +// replacing the dictionary page with an OnPair blob, since both ask the same +// question of the spec. It is stored in the header rather than threaded through +// the decode lambda so the payload stays self-describing; the word it occupies +// was already there as padding, so the RLE_DICTIONARY sizes do not move. +constexpr size_t kDictHeader = 16; + +std::vector BuildDictPayload(const Corpus& c, + const std::vector& vals, + const parquet::ColumnDescriptor* descr, + parquet::Encoding::type dict_page_enc) { + auto base = parquet::MakeEncoder(parquet::Type::BYTE_ARRAY, parquet::Encoding::PLAIN, + /*use_dictionary=*/true, descr); + auto* enc = dynamic_cast*>(base.get()); + auto* dict_enc = dynamic_cast*>(base.get()); + enc->Put(vals.data(), static_cast(c.n_rows())); + + size_t dict_bytes = static_cast(dict_enc->dict_encoded_size()); + std::vector dict_page(dict_bytes); + dict_enc->WriteDict(dict_page.data()); + int num_entries = dict_enc->num_entries(); + + std::vector idx(static_cast(enc->EstimatedDataEncodedSize()) + 16); + int idx_bytes = dict_enc->WriteIndices(idx.data(), static_cast(idx.size())); + if (idx_bytes <= 0) { + std::fprintf(stderr, "RLE_DICTIONARY: WriteIndices failed\n"); + std::abort(); + } + + // Re-encode the dictionary entries under another encoding. The detour through + // PLAIN is unavoidable: WriteDict is the only public way to get the entries in + // id order, and that order has to survive or the index stream stops matching. + // It makes this variant's *encode* throughput pessimistic -- a writer for such + // a format would encode the entries once -- but leaves ratio and decode exact. + if (dict_page_enc != parquet::Encoding::PLAIN) { + auto pd = parquet::MakeTypedDecoder(parquet::Encoding::PLAIN); + pd->SetData(num_entries, dict_page.data(), static_cast(dict_bytes)); + std::vector entries(num_entries); + if (pd->Decode(entries.data(), num_entries) != num_entries) { + std::fprintf(stderr, "dictionary page re-encode: short read\n"); + std::abort(); + } + auto re = parquet::MakeTypedEncoder(dict_page_enc); + re->Put(entries.data(), num_entries); + auto buf = re->FlushValues(); + dict_page.assign(buf->data(), buf->data() + buf->size()); + dict_bytes = dict_page.size(); + } + + std::vector v; + v.reserve(kDictHeader + dict_bytes + static_cast(idx_bytes)); + PutU32(&v, static_cast(dict_bytes)); + PutU32(&v, static_cast(idx_bytes)); + PutU32(&v, static_cast(num_entries)); + PutU32(&v, static_cast(dict_page_enc)); + v.insert(v.end(), dict_page.begin(), dict_page.end()); + v.insert(v.end(), idx.begin(), idx.begin() + idx_bytes); + return v; +} + +size_t DecodeDictPayload(const Corpus& c, const parquet::ColumnDescriptor* descr, + const uint8_t* p, size_t /*size*/, uint8_t* out) { + size_t dict_bytes = GetU32(p); + size_t idx_bytes = GetU32(p + 4); + int num_entries = static_cast(GetU32(p + 8)); + auto dict_page_enc = static_cast(GetU32(p + 12)); + const uint8_t* dict_page = p + kDictHeader; + const uint8_t* idx = dict_page + dict_bytes; + + // SetDict below takes any TypedDecoder, so the dictionary page's encoding is + // free to vary while the index path stays identical. + auto dict_dec = parquet::MakeTypedDecoder(dict_page_enc); + dict_dec->SetData(num_entries, dict_page, static_cast(dict_bytes)); + + int n = static_cast(c.n_rows()); + auto dec = parquet::MakeDictDecoder(descr); + dec->SetDict(dict_dec.get()); + dec->SetData(n, idx, static_cast(idx_bytes)); + std::vector vals(n); + int got = dec->Decode(vals.data(), n); + if (got != n) { + std::fprintf(stderr, "RLE_DICTIONARY decode short read (%d of %d)\n", got, n); + std::abort(); + } + return Gather(vals, out); +} + +// zstd / lz4 over the concatenated value bytes, in two accountings. +// +// The companion benchmark charges zstd/lz4 a *separate, uncompressed* bit-packed +// row-length array, because a raw frame of concatenated bytes cannot recover row +// boundaries on its own. A real Parquet page puts the lengths inside the page, so +// the page compressor sees them too -- and on columns of near-constant width that +// array is almost free once compressed. Both are reported here: the split-length +// figure is what Tables 1-3 print, the in-page figure is what Parquet stores. + +std::vector BuildConcatPayload(const Corpus& c, bool with_lengths) { + std::vector v = c.bytes; + if (with_lengths) AppendLengths(c, &v); + return v; +} + +} // namespace + +int main(int argc, char** argv) { + std::string dir = CorpusDir(argc, argv); + std::vector files = CorpusFiles(dir); + if (files.empty()) { + std::fprintf(stderr, "no .txt corpora in %s\n", dir.c_str()); + return 1; + } + auto descr = ByteArrayDescr(); + + std::printf("%-30s %10s %10s %7s %9s %9s\n", "corpus", "rows", "raw MiB", "ratio", + "enc MiB/s", "dec MiB/s"); + std::printf("%s\n", std::string(92, '-').c_str()); + + for (const auto& path : files) { + Corpus c = ReadCorpus(path); + double threshold = ThresholdFor(c.name); + std::vector vals = CorpusValues(c); + const size_t out_cap = c.raw_bytes() + op::kDecodePadding + kPad; + + op::Config cfg{PickOnPairBits(c, threshold), threshold, 42}; + op::Config cfg16{16, threshold, 42}; + op::Config cfg_dop{PickDictOnPairBits(c, threshold), threshold, 42}; + std::vector dop_scratch; + + auto fsst_build = [&] { return BuildFsstPayload(c); }; + auto fsst_decode = [&](const uint8_t* p, size_t s, uint8_t* o) { + return DecodeFsstPayload(p, s, o, out_cap); + }; + auto op_build = [&] { return BuildOnPairPayload(c, cfg, /*byte_aligned=*/false); }; + auto op16_build = [&] { return BuildOnPairPayload(c, cfg16, /*byte_aligned=*/false); }; + auto opba_build = [&] { return BuildOnPairPayload(c, cfg, /*byte_aligned=*/true); }; + auto op_decode = [&](const uint8_t* p, size_t s, uint8_t* o) { + return DecodeOnPairPayload(p, s, o); + }; + auto dop_build = [&] { return BuildDictOnPairPayload(c, cfg16); }; + auto dop_auto_build = [&] { return BuildDictOnPairPayload(c, cfg_dop); }; + auto dop_decode = [&](const uint8_t* p, size_t /*s*/, uint8_t* o) { + return DecodeDictOnPairPayload(p, o, &dop_scratch); + }; + auto concat_build = [&] { return BuildConcatPayload(c, /*with_lengths=*/true); }; + auto bytes_build = [&] { return BuildConcatPayload(c, /*with_lengths=*/false); }; + auto concat_decode = [&](const uint8_t* p, size_t /*s*/, uint8_t* o) { + std::memcpy(o, p, c.raw_bytes()); + return c.raw_bytes(); + }; + auto pq = [&](parquet::Encoding::type e) { + return std::make_pair(BuildFn([&, e] { return BuildPqPayload(c, vals, e); }), + DecodeFn([&, e](const uint8_t* p, size_t s, uint8_t* o) { + return DecodePqPayload(c, e, p, s, o); + })); + }; + auto dict_of = [&](parquet::Encoding::type dpe) { + return std::make_pair( + BuildFn([&, dpe] { return BuildDictPayload(c, vals, descr.get(), dpe); }), + DecodeFn([&](const uint8_t* p, size_t s, uint8_t* o) { + return DecodeDictPayload(c, descr.get(), p, s, o); + })); + }; + auto dict_plain = dict_of(parquet::Encoding::PLAIN); + auto dict_dlba = dict_of(parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY); + const BuildFn& dict_build = dict_plain.first; + const DecodeFn& dict_decode = dict_plain.second; + + std::vector ms; + // Reference points, measured here so every ratio in this table shares one + // accounting. The two [split len] rows reproduce the companion benchmark's + // zstd/lz4 columns exactly (uncompressed row-length array charged on the + // side); the two [len in page] rows are the same codecs with the lengths + // inside the compressed page, which is what Parquet actually writes. FSST + // and OnPair alone already match Tables 1-3, since their payload carries the + // bit-packed length array uncompressed either way. + ms.push_back(RunCodec(c, "zstd(1) [split len]", Generic::kZstd1, bytes_build, concat_decode, + c.len_array_bytes())); + ms.push_back(RunCodec(c, "lz4 [split len]", Generic::kLz4, bytes_build, concat_decode, + c.len_array_bytes())); + ms.push_back( + RunCodec(c, "zstd(1) [len in page]", Generic::kZstd1, concat_build, concat_decode)); + ms.push_back(RunCodec(c, "lz4 [len in page]", Generic::kLz4, concat_build, concat_decode)); + ms.push_back(RunCodec(c, "FSST", Generic::kNone, fsst_build, fsst_decode)); + ms.push_back(RunCodec(c, "OnPair-auto", Generic::kNone, op_build, op_decode)); + ms.push_back(RunCodec(c, "OnPair16", Generic::kNone, op16_build, op_decode)); + ms.push_back(RunCodec(c, "DICT+OnPair", Generic::kNone, dop_build, dop_decode)); + ms.push_back(RunCodec(c, "DICT+OnPair-auto", Generic::kNone, dop_auto_build, dop_decode)); + + // Group A: cascade a generic codec over FSST / OnPair. + ms.push_back(RunCodec(c, "FSST+zstd(1)", Generic::kZstd1, fsst_build, fsst_decode)); + ms.push_back(RunCodec(c, "FSST+lz4", Generic::kLz4, fsst_build, fsst_decode)); + ms.push_back(RunCodec(c, "OnPair-auto+zstd(1)", Generic::kZstd1, op_build, op_decode)); + ms.push_back(RunCodec(c, "OnPair-auto+lz4", Generic::kLz4, op_build, op_decode)); + ms.push_back(RunCodec(c, "OnPair-bytealign", Generic::kNone, opba_build, op_decode)); + ms.push_back( + RunCodec(c, "OnPair-bytealign+zstd(1)", Generic::kZstd1, opba_build, op_decode)); + + // Group B: Parquet's own byte-array encodings, alone and cascaded. + struct PqCase { + const char* label; + parquet::Encoding::type enc; + }; + for (const PqCase& pc : + {PqCase{"PLAIN", parquet::Encoding::PLAIN}, + PqCase{"DELTA_LENGTH_BYTE_ARRAY", parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY}, + PqCase{"DELTA_BYTE_ARRAY", parquet::Encoding::DELTA_BYTE_ARRAY}}) { + auto fns = pq(pc.enc); + for (Generic g : {Generic::kNone, Generic::kZstd1, Generic::kLz4}) { + ms.push_back(RunCodec(c, pc.label + std::string(GenericSuffix(g)), g, fns.first, + fns.second)); + } + } + for (Generic g : {Generic::kNone, Generic::kZstd1, Generic::kLz4}) { + ms.push_back(RunCodec(c, "RLE_DICTIONARY" + std::string(GenericSuffix(g)), g, dict_build, + dict_decode)); + } + // A dictionary page carrying DELTA_LENGTH_BYTE_ARRAY instead of PLAIN. Not a + // page any writer can emit -- the spec fixes BYTE_ARRAY dictionary pages at + // PLAIN -- so these two rows measure a format change, and are the opponent + // DICT+OnPair deserves: both replace the dictionary page's encoding and leave + // the index stream alone. Encode throughput here is pessimistic; see + // BuildDictPayload. + for (Generic g : {Generic::kNone, Generic::kZstd1}) { + ms.push_back(RunCodec(c, "DICT+DLBA" + std::string(GenericSuffix(g)), g, dict_dlba.first, + dict_dlba.second)); + } + + std::printf("%-30s %10zu %10.2f\n", c.name.c_str(), c.n_rows(), Mib(c.raw_bytes())); + auto ratio = [&](const Measured& m) { + return static_cast(c.raw_bytes()) / static_cast(m.compressed_bytes); + }; + for (const Measured& m : ms) { + std::printf(" %-32s %10.2f %8.3fx %9.1f %9.1f\n", m.label.c_str(), + Mib(m.compressed_bytes), ratio(m), m.encode_mibs, m.decode_mibs); + } + + // The two questions this benchmark exists to answer, stated per corpus: + // what the cascade buys over the encoding alone, and how the best + // Parquet-native page compares with OnPair alone. + auto find = [&](const std::string& label) -> const Measured* { + for (const Measured& m : ms) + if (m.label == label) return &m; + return nullptr; + }; + const Measured* fsst_alone = find("FSST"); + const Measured* fsst_z = find("FSST+zstd(1)"); + const Measured* op_alone = find("OnPair-auto"); + const Measured* op_z = find("OnPair-auto+zstd(1)"); + const Measured* opba_z = find("OnPair-bytealign+zstd(1)"); + std::printf(" -> FSST+zstd vs FSST alone: ratio %+.1f%%, decode %+.1f%%\n", + (ratio(*fsst_z) / ratio(*fsst_alone) - 1.0) * 100.0, + (fsst_z->decode_mibs / fsst_alone->decode_mibs - 1.0) * 100.0); + std::printf(" -> OnPair+zstd vs OnPair alone: ratio %+.1f%%, decode %+.1f%%\n", + (ratio(*op_z) / ratio(*op_alone) - 1.0) * 100.0, + (op_z->decode_mibs / op_alone->decode_mibs - 1.0) * 100.0); + std::printf(" -> OnPair byte-aligned+zstd vs bit-packed OnPair alone: ratio %+.1f%%\n", + (ratio(*opba_z) / ratio(*op_alone) - 1.0) * 100.0); + + const Measured* best_pq = nullptr; + for (const Measured& m : ms) { + if (m.label.rfind("PLAIN", 0) != 0 && m.label.rfind("DELTA", 0) != 0 && + m.label.rfind("RLE_DICTIONARY", 0) != 0) { + continue; + } + if (best_pq == nullptr || m.compressed_bytes < best_pq->compressed_bytes) best_pq = &m; + } + std::printf(" -> best Parquet-native (%s): %.3fx; OnPair alone %+.1f%% vs it\n\n", + best_pq->label.c_str(), ratio(*best_pq), + (ratio(*op_alone) / ratio(*best_pq) - 1.0) * 100.0); + std::fflush(stdout); + } + return 0; +} From 86acb3e752895a409585ccbf5a802057696db72f Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 3 Aug 2026 06:02:20 +0000 Subject: [PATCH 21/24] Verify every code width, and isolate width from decode speed The roundtrip checker now runs every dictionary budget from 9 to 16 rather than only 16, for both the plain and the dedup layout, and exits non-zero on any mismatch so it can gate a benchmark run. The packed decode loop is templated on the code width and the auto budget picks a width per column, so checking only 16 left the width the benchmarks report unverified. The width sweep answers whether a wider code decodes faster without the confound a budget sweep carries. It holds one trained dictionary and its code stream fixed and re-packs the same codes at every width up to 16, so tokens, token count and copy width are identical and only the unpacking differs. --- cpp/src/parquet/onpair/verify_roundtrip.cc | 212 ++++++++++++++ .../parquet/onpair/width_sweep_benchmark.cc | 262 ++++++++++++++++++ 2 files changed, 474 insertions(+) create mode 100644 cpp/src/parquet/onpair/verify_roundtrip.cc create mode 100644 cpp/src/parquet/onpair/width_sweep_benchmark.cc diff --git a/cpp/src/parquet/onpair/verify_roundtrip.cc b/cpp/src/parquet/onpair/verify_roundtrip.cc new file mode 100644 index 000000000000..621e9fd60ffe --- /dev/null +++ b/cpp/src/parquet/onpair/verify_roundtrip.cc @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Visible round-trip proof for the OnPair port: decode both plain OnPair16 and +// OnPair16-dedup and check EVERY row equals the original bytes, then print a few +// concrete original -> decoded samples so a human can eyeball the recovery. +// +// Every dictionary budget in the valid 9..16 range is checked, not just 16: the +// packed decode loop is templated on the code width, and OnPair-auto picks a +// width per column, so verifying only 16 would leave the width the benchmarks +// actually report unverified. The merge threshold comes from bench_common.h so +// the dictionary trained here is the one the benchmarks measure. +// +// Exits non-zero if any row of any corpus at any width fails to round-trip, so +// this can gate a run. +// +// Build (one line): g++ -std=c++17 -O2 -Icpp/src +// cpp/src/parquet/onpair/onpair.cc cpp/src/parquet/onpair/verify_roundtrip.cc -o /tmp/verify +// Run: /tmp/verify bench-fsst-onpair/corpora/tpch_l_shipmode.txt [more files...] + +#include +#include +#include +#include +#include +#include +#include + +#include "parquet/onpair/bench_common.h" +#include "parquet/onpair/onpair.h" + +namespace op = parquet::onpair; + +namespace { + +struct Corpus { + std::vector bytes; + std::vector offsets; + size_t rows() const { return offsets.size() - 1; } + std::string row(size_t i) const { + return std::string(reinterpret_cast(bytes.data() + offsets[i]), + offsets[i + 1] - offsets[i]); + } +}; + +Corpus Read(const char* path) { + Corpus c; + std::ifstream in(path, std::ios::binary); + c.offsets.push_back(0); + std::string line; + while (std::getline(in, line)) { + c.bytes.insert(c.bytes.end(), line.begin(), line.end()); + c.offsets.push_back(static_cast(c.bytes.size())); + } + return c; +} + +// Compare a decoded concatenated buffer to the original, row by row. +// Returns the number of mismatching rows and the first mismatching index. +size_t CheckPerRow(const Corpus& c, const uint8_t* decoded, size_t dn, long* first_bad) { + *first_bad = -1; + if (dn != c.bytes.size()) { + *first_bad = 0; + return c.rows(); + } + size_t bad = 0; + for (size_t i = 0; i < c.rows(); ++i) { + size_t off = c.offsets[i], len = c.offsets[i + 1] - off; + if (std::memcmp(decoded + off, c.bytes.data() + off, len) != 0) { + if (*first_bad < 0) *first_bad = static_cast(i); + ++bad; + } + } + return bad; +} + +std::string Trunc(const std::string& s, size_t n = 42) { + return s.size() <= n ? s : s.substr(0, n) + "…"; +} + +void Samples(const Corpus& c, const uint8_t* decoded) { + size_t r = c.rows(); + size_t idx[4] = {0, r / 3, (2 * r) / 3, r - 1}; + for (size_t k = 0; k < 4; ++k) { + size_t i = idx[k]; + size_t off = c.offsets[i], len = c.offsets[i + 1] - off; + std::string dec(reinterpret_cast(decoded + off), len); + std::string orig = c.row(i); + std::printf(" row %-8zu original=%-44s decoded=%-44s %s\n", i, + ("\"" + Trunc(orig) + "\"").c_str(), ("\"" + Trunc(dec) + "\"").c_str(), + orig == dec ? "MATCH" : "*** MISMATCH ***"); + } +} + +// OnPair (no dedup): compress then whole-column decode. +bool VerifyOnPair(const Corpus& c, uint8_t bits, double threshold, bool show_samples) { + op::Config cfg{bits, threshold, 42}; + op::Column col = op::Compress(c.bytes.data(), c.bytes.size(), c.offsets.data(), c.rows(), cfg); + std::vector out(op::DecodedLen(col) + op::kDecodePadding, 0); + size_t dn = op::DecompressInto(col, out.data()); + long bad_at; + size_t bad = CheckPerRow(c, out.data(), dn, &bad_at); + std::printf(" OnPair%-2u : %zu/%zu rows exact %s\n", bits, c.rows() - bad, c.rows(), + bad == 0 ? "[OK]" : "[FAIL]"); + if (bad != 0) std::printf(" first mismatching row: %ld\n", bad_at); + if (show_samples) Samples(c, out.data()); + return bad == 0; +} + +// The distinct-value set a dedup cascade OnPairs, plus the per-row ids into it. +// Built once per corpus and reused across widths -- deduplicating 500k rows is +// far more expensive than the training pass being verified. +struct Distinct { + std::vector bytes; + std::vector offsets{0}; + std::vector refs; + size_t count() const { return offsets.size() - 1; } +}; + +Distinct BuildDistinct(const Corpus& c) { + size_t n = c.rows(); + Distinct d; + d.refs.resize(n); + std::unordered_map ids; + for (size_t i = 0; i < n; ++i) { + std::string v = c.row(i); + auto it = ids.find(v); + uint32_t id; + if (it == ids.end()) { + id = static_cast(ids.size()); + d.bytes.insert(d.bytes.end(), v.begin(), v.end()); + d.offsets.push_back(static_cast(d.bytes.size())); + ids.emplace(std::move(v), id); + } else { + id = it->second; + } + d.refs[i] = id; + } + return d; +} + +// OnPair-dedup: OnPair the distinct values, then gather every row back through +// its id. Checks the gather as well as the decode -- a correct dictionary paired +// with a broken reference column still reconstructs the wrong column. +bool VerifyOnPairDedup(const Corpus& c, const Distinct& d, uint8_t bits, double threshold, + bool show_samples) { + op::Config cfg{bits, threshold, 42}; + op::Column col = op::Compress(d.bytes.data(), d.bytes.size(), d.offsets.data(), d.count(), cfg); + std::vector dbuf(op::DecodedLen(col) + op::kDecodePadding, 0); + op::DecompressInto(col, dbuf.data()); // distinct values, concatenated in id order + std::vector out(c.bytes.size() + 16, 0); + size_t w = 0; + for (size_t i = 0; i < c.rows(); ++i) { + uint32_t id = d.refs[i]; + size_t off = d.offsets[id], len = d.offsets[id + 1] - off; + std::memcpy(out.data() + w, dbuf.data() + off, len); + w += len; + } + long bad_at; + size_t bad = CheckPerRow(c, out.data(), w, &bad_at); + std::printf(" OnPair%-2u-dedup : %zu/%zu rows exact (%zu distinct) %s\n", bits, + c.rows() - bad, c.rows(), d.count(), bad == 0 ? "[OK]" : "[FAIL]"); + if (bad != 0) std::printf(" first mismatching row: %ld\n", bad_at); + if (show_samples) Samples(c, out.data()); + return bad == 0; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 2) { + std::fprintf(stderr, "usage: %s [more...]\n", argv[0]); + return 2; + } + int failures = 0; + size_t rows_checked = 0; + for (int a = 1; a < argc; ++a) { + Corpus c = Read(argv[a]); + if (c.rows() == 0) { + std::fprintf(stderr, "%s: no rows read\n", argv[a]); + ++failures; + continue; + } + // Same rule the benchmarks use, so the trained dictionary matches theirs. + double threshold = bench::ThresholdFor(std::filesystem::path(argv[a]).stem().string()); + std::printf("\n%s (%zu rows, %.2f MiB, threshold %.2f)\n", argv[a], c.rows(), + c.bytes.size() / (1024.0 * 1024.0), threshold); + Distinct d = BuildDistinct(c); + for (uint8_t bits = 9; bits <= 16; ++bits) { + // Samples are the human-readable proof; print them once, at the width the + // report's fixed-budget rows use, rather than eight times per corpus. + bool show = (bits == 16); + if (!VerifyOnPair(c, bits, threshold, show)) ++failures; + if (!VerifyOnPairDedup(c, d, bits, threshold, show)) ++failures; + rows_checked += 2 * c.rows(); + } + } + std::printf("\n%zu row comparisons, %d failure(s)\n", rows_checked, failures); + return failures == 0 ? 0 : 1; +} diff --git a/cpp/src/parquet/onpair/width_sweep_benchmark.cc b/cpp/src/parquet/onpair/width_sweep_benchmark.cc new file mode 100644 index 000000000000..e1956f4c41c8 --- /dev/null +++ b/cpp/src/parquet/onpair/width_sweep_benchmark.cc @@ -0,0 +1,262 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with this +// work for additional information regarding copyright ownership. The ASF +// licenses this file to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Does a wider code make OnPair decode faster? Isolate the code width from +// everything else that changes with the dictionary budget. +// +// The obvious way to ask this -- compare OnPair-auto (which picks a budget per +// column) against OnPair16 -- cannot answer it. A narrower budget trains a +// SMALLER DICTIONARY, so it also changes the tokens, the token count per row, and +// the gather-copy width the decoder picks from max_token_len. Those move decode +// far more than the unpacking does, and they move in both directions, so the +// comparison is confounded and its answer is noise. +// +// This benchmark holds the dictionary fixed and varies only the packing width. +// For each training budget it takes the ONE trained dictionary and its ONE code +// stream, then bit-packs those same codes at every width from their true width up +// to 16 and times DecompressPacked at each. Identical tokens, identical code +// sequence, identical output bytes, identical copy width -- the only difference is +// how many bits each code occupies and which DecompressPackedFixedBits<> template +// the dispatch lands on. Storing a code in more bits than it needs is pure waste +// on the ratio axis, so any decode gain is the whole case for a wider code. +// +// It also prints the confounded comparison alongside, so the two can be read +// against each other: the "own width" column across training budgets is what a +// budget sweep sees, and the widen-in-place rows are what the width alone does. +// +// PIN AND QUIESCE: this is a timing benchmark. Run it under `taskset -c 0` with +// core 0 idle -- `ps -eo pid,psr,pcpu | awk '$2==0 && $3>5'` must print nothing. +// +// Build (one line): +// g++ -std=c++17 -O3 -march=native -Icpp/src \ +// cpp/src/parquet/onpair/onpair.cc \ +// cpp/src/parquet/onpair/width_sweep_benchmark.cc \ +// /usr/lib64/libzstd.so.1 /usr/lib64/liblz4.so.1 -o /tmp/width_sweep +// Run: +// taskset -c 0 /tmp/width_sweep /tmp/tmp/corpora30 + +#include +#include +#include +#include +#include + +#include "parquet/onpair/bench_common.h" +#include "parquet/onpair/onpair.h" + +namespace op = parquet::onpair; + +namespace { + +// Training budgets to build a dictionary at. Not all of 9..16: each one is a full +// training pass, and three points spanning the range (floor, middle, ceiling) show +// whether the width effect depends on dictionary size. The widths swept per +// dictionary are exhaustive, since that is the axis under test. +constexpr uint8_t kBudgets[] = {9, 12, 16}; + +struct WidthPoint { + size_t bits = 0; + double decode_mibs = 0; + size_t codes_bytes = 0; // bit-packed code stream, logical size +}; + +struct BudgetResult { + uint8_t budget = 0; + size_t num_tokens = 0; + size_t true_bits = 0; // ceil(log2 num_tokens) -- the width a real page stores + size_t num_codes = 0; + size_t max_token_len = 0; + double bytes_per_token = 0; + std::vector widths; + + const WidthPoint* at(size_t bits) const { + for (const WidthPoint& w : widths) { + if (w.bits == bits) return &w; + } + return nullptr; + } +}; + +// Time DecompressPacked on one (dictionary, code stream, width), the same way +// RunCodec does: fresh output buffer per iteration allocated outside the timed +// region, median of kDecodeIters, throughput over the RAW bytes. Aborts if the +// decode does not reproduce the column exactly -- a width that unpacks to the +// wrong codes would otherwise read as a fast decode. +double TimeDecode(const bench::Corpus& c, const op::CompactDictionary& dict, + const std::vector& packed, size_t num_codes, size_t bits) { + const size_t out_cap = c.raw_bytes() + op::kDecodePadding + 64; + { + std::vector out(out_cap, 0); + size_t w = op::DecompressPacked(dict, packed.data(), num_codes, bits, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "width %zu roundtrip mismatch on %s (w=%zu raw=%zu)\n", bits, + c.name.c_str(), w, c.raw_bytes()); + std::abort(); + } + } + std::vector mibs; + for (int it = 0; it < bench::kDecodeIters; ++it) { + std::vector out(out_cap, 0); + auto t0 = bench::Clock::now(); + size_t w = op::DecompressPacked(dict, packed.data(), num_codes, bits, out.data()); + double dt = std::chrono::duration(bench::Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + mibs.push_back(bench::Mib(c.raw_bytes()) / dt); + } + return bench::Median(std::move(mibs)); +} + +BudgetResult SweepBudget(const bench::Corpus& c, uint8_t budget, double threshold) { + op::Config cfg{budget, threshold, 42}; + op::Column col = + op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), c.n_rows(), cfg); + + BudgetResult r; + r.budget = budget; + r.num_tokens = col.dict.num_tokens(); + r.true_bits = bench::IndexBits(r.num_tokens); + r.num_codes = col.codes.size(); + r.max_token_len = col.dict.max_token_len; + r.bytes_per_token = static_cast(c.raw_bytes()) / static_cast(r.num_codes); + + // Widen once; PackValues takes u32. The same values are re-packed at each width, + // so every width decodes the identical code sequence. + std::vector codes32(col.codes.begin(), col.codes.end()); + + for (size_t bits = r.true_bits; bits <= 16; ++bits) { + std::vector packed = op::PackValues(codes32.data(), codes32.size(), bits); + WidthPoint p; + p.bits = bits; + p.codes_bytes = bench::BitPackedBytes(r.num_codes, bits); + p.decode_mibs = TimeDecode(c, col.dict, packed, r.num_codes, bits); + r.widths.push_back(p); + } + return r; +} + +void PrintCorpus(const bench::Corpus& c, const std::vector& results) { + std::printf("\n%s %zu rows, %.2f MiB raw\n", c.name.c_str(), c.n_rows(), + bench::Mib(c.raw_bytes())); + for (const BudgetResult& r : results) { + std::printf( + " budget %2ub: %6zu tokens, true width %zu b, %zu codes (%.2f raw B/token), " + "copy width %zu\n", + r.budget, r.num_tokens, r.true_bits, r.num_codes, r.bytes_per_token, r.max_token_len); + std::printf(" width :"); + for (const WidthPoint& w : r.widths) std::printf(" %8zu", w.bits); + std::printf("\n MiB/s :"); + for (const WidthPoint& w : r.widths) std::printf(" %8.0f", w.decode_mibs); + std::printf("\n vs true:"); + const WidthPoint* base = r.at(r.true_bits); + for (const WidthPoint& w : r.widths) { + std::printf(" %+7.1f%%", 100.0 * (w.decode_mibs - base->decode_mibs) / base->decode_mibs); + } + std::printf("\n codes :"); + for (const WidthPoint& w : r.widths) { + std::printf(" %+7.1f%%", + 100.0 * (static_cast(w.codes_bytes) - base->codes_bytes) / + base->codes_bytes); + } + std::printf(" (bit-packed code stream, vs true width)\n"); + } +} + +} // namespace + +int main(int argc, char** argv) { + std::vector files = bench::CorpusFiles(bench::CorpusDir(argc, argv)); + if (files.empty()) { + std::fprintf(stderr, "no .txt corpora found\n"); + return 2; + } + std::printf( + "OnPair decode throughput vs CODE WIDTH ALONE -- one trained dictionary per\n" + "budget, its code stream re-packed at each width from its true width to 16.\n" + "Same tokens, same code sequence, same copy width; only the unpacking differs.\n" + "'codes' is what the wider packing costs on the ratio axis.\n" + "%d decode iterations, median. Pin to an idle core.\n", + bench::kDecodeIters); + + // Per-corpus deltas for the summary: going from the true width to 16 bits, and + // to true+1, on each dictionary. + struct Delta { + std::string corpus; + uint8_t budget; + size_t true_bits; + double to_16; + double to_plus1; + double codes_cost_16; + }; + std::vector deltas; + + for (const std::filesystem::path& f : files) { + bench::Corpus c = bench::ReadCorpus(f); + if (c.n_rows() == 0) { + std::fprintf(stderr, "%s: no rows\n", f.c_str()); + return 1; + } + double threshold = bench::ThresholdFor(c.name); + std::vector results; + for (uint8_t b : kBudgets) results.push_back(SweepBudget(c, b, threshold)); + PrintCorpus(c, results); + std::fflush(stdout); + + for (const BudgetResult& r : results) { + if (r.true_bits >= 16) continue; // nothing to widen into + const WidthPoint* base = r.at(r.true_bits); + const WidthPoint* w16 = r.at(16); + const WidthPoint* wp1 = r.at(r.true_bits + 1); + deltas.push_back({c.name, r.budget, r.true_bits, + 100.0 * (w16->decode_mibs - base->decode_mibs) / base->decode_mibs, + 100.0 * (wp1->decode_mibs - base->decode_mibs) / base->decode_mibs, + 100.0 * (static_cast(w16->codes_bytes) - base->codes_bytes) / + base->codes_bytes}); + } + } + + // Summary. The question is whether widening the code buys decode speed, so the + // headline is the sign and size of the true-width -> 16-bit change, against what + // that widening costs on the code stream. + std::printf("\n\n=== Summary: decode change from widening the code, dictionary held fixed ===\n"); + std::printf("%-30s %6s %6s %10s %10s %12s\n", "corpus", "budget", "true b", "->true+1", + "->16 b", "codes at 16b"); + std::vector all16, allp1, cost16; + for (const Delta& d : deltas) { + std::printf("%-30s %5ub %5zub %+9.1f%% %+9.1f%% %+11.1f%%\n", d.corpus.c_str(), d.budget, + d.true_bits, d.to_plus1, d.to_16, d.codes_cost_16); + all16.push_back(d.to_16); + allp1.push_back(d.to_plus1); + cost16.push_back(d.codes_cost_16); + } + if (!all16.empty()) { + auto stats = [](std::vector v, const char* label) { + std::sort(v.begin(), v.end()); + int faster = 0; + for (double x : v) { + if (x > 0) ++faster; + } + std::printf(" %-22s median %+6.1f%% min %+6.1f%% max %+6.1f%% faster on %d/%zu\n", label, + bench::Median(v), v.front(), v.back(), faster, v.size()); + }; + std::printf("\n%zu (corpus, budget) pairs where the true width is below 16:\n", all16.size()); + stats(allp1, "decode, true -> true+1"); + stats(all16, "decode, true -> 16 b"); + std::vector c16 = cost16; + std::sort(c16.begin(), c16.end()); + std::printf(" %-22s median %+6.1f%% min %+6.1f%% max %+6.1f%%\n", "code stream at 16 b", + bench::Median(c16), c16.front(), c16.back()); + } + return 0; +} From ab5f6d6dac858e6f407f523ff710501a2f9f4ac8 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 3 Aug 2026 21:31:53 +0000 Subject: [PATCH 22/24] onpair: decode through a strided dictionary view, not the stored blob The comment above the decode loop claimed it was store-bandwidth-bound, on the evidence that (over-copy factor) x (decode MiB/s) was constant across corpora. That product is equally constant when the loop is bound by tokens retired, since over-copy is kCopy/mean_token_len and throughput is mean_token_len x tokens/s -- the two models are indistinguishable from that measurement, and the wrong one was picked. It then steered two experiments (block-wise unpacking with prefetch, a narrower two-store copy) at a cost that was not the constraint. An ablation ladder separates them. Deleting the stores makes the loop slower; deleting the random dictionary read nearly doubles it. The cost is the gather: the stored layout reads a u32 offsets pair from one array and the payload from a variable-stride blob, so every token touches two independent random cache lines. FSST's decoder never pays this -- it reads a fixed-stride symbol[] plus len[]. Add StridedDictionary, a decode-side view giving each token a fixed 16-byte slot and a length byte in a dense side array, so a token is one cache line. It is deliberately not the stored form: ~17 bytes per token against ~12 would add hundreds of KiB to a 65k-token dictionary and move every ratio. It is built once per column from an unchanged CompactDictionary and thrown away, costing nothing on disk. Skipped when the code stream is too short to amortise the O(tokens) build. Where a predicated store is available the loop writes exactly one token's length, which removes the fixed over-copy and collapses the 4/8/16 width dispatch to a single kernel per code width. The guard is compile-time on purpose: the portable path is byte-identical and only slower, so a build without it loses speed and nothing else. The strided load alone, with today's fixed-width store, is worth most of the win and needs no ISA feature. DecompressPacked keeps its signature and builds the view internally; a new overload takes a view the caller built once, for a reader decoding many pages against one dictionary. verify_roundtrip gains a byte-exact gate gating all four decode paths -- whole column, self-building packed, prebuilt view, and the short-stream fallback -- against a scalar reference, since nothing else reaches the last of those. Verified under -march=native and the portable fallback: 30 corpora, 86,400,060 row comparisons, 120 path gates, no disagreements under either. Compressed sizes and ratios are bit-identical to the published ladder run on all shared rows. --- cpp/src/parquet/onpair/onpair.cc | 315 +++++++++++++++++---- cpp/src/parquet/onpair/onpair.h | 44 +++ cpp/src/parquet/onpair/verify_roundtrip.cc | 75 ++++- 3 files changed, 379 insertions(+), 55 deletions(-) diff --git a/cpp/src/parquet/onpair/onpair.cc b/cpp/src/parquet/onpair/onpair.cc index 2cd99c100d0c..af5882654aaf 100644 --- a/cpp/src/parquet/onpair/onpair.cc +++ b/cpp/src/parquet/onpair/onpair.cc @@ -22,6 +22,14 @@ #include #include +// The decode loop stores exactly one token's length per iteration when a +// predicated store is available, which removes the fixed over-copy entirely. This +// is a compile-time guard on purpose: the portable path below is byte-identical +// and only slower, so a build without SVE loses speed and nothing else. +#if defined(__ARM_FEATURE_SVE) +#include +#endif + namespace parquet::onpair { namespace { @@ -734,10 +742,74 @@ size_t DecodedLen(const Column& col) { return sum; } +void StridedDictionary::Build(const CompactDictionary& dict) { + // Align `slots` to a cache line so that a kStride-byte slot, kStride being a + // power of two no larger than a line, never straddles two lines. std::vector + // only promises alignment for its element type, so over-allocate by one line + // and point into `storage` at the first aligned byte. + constexpr size_t kAlign = 64; + const size_t ntokens = dict.num_tokens(); + const size_t slot_bytes = (ntokens * kStride + kAlign - 1) / kAlign * kAlign; + storage.assign(slot_bytes + kAlign, 0); + const size_t misalign = reinterpret_cast(storage.data()) % kAlign; + slots = storage.data() + (misalign == 0 ? 0 : kAlign - misalign); + lens.resize(ntokens); + for (size_t t = 0; t < ntokens; ++t) { + // Tokens are capped at kMaxTokenSize == kStride by training, so a token fills + // at most its own slot and a length always fits in a byte. The zero-fill above + // is what makes the unused tail of a slot well-defined for a fixed-width read. + const size_t len = dict.offsets[t + 1] - dict.offsets[t]; + std::memcpy(slots + t * kStride, dict.bytes.data() + dict.offsets[t], len); + lens[t] = static_cast(len); + } + max_token_len = dict.max_token_len; +} + namespace { -// Shared body of DecompressInto, parameterised on the gather-copy width for the -// same reason DecompressPackedFixed is. See CompactDictionary::max_token_len. +// Building the strided view is one pass over the dictionary, so it pays for itself +// only when the code stream is long enough to amortise it. Below this the blob +// kernel runs directly. The crossover measured at roughly one code per token; the +// build-charged rung's worst corpus (a two-token dictionary over a short column) +// sits exactly here and came out level rather than slower. +bool StridedViewWorthBuilding(size_t ncodes, size_t ntokens) { return ncodes >= ntokens; } + +// Shared body of DecompressInto over the strided view, parameterised on the copy +// width for the same reason the packed kernels are. See max_token_len. +template +size_t DecompressIntoStrided(const StridedDictionary& dict, const Column& col, uint8_t* out) { + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + size_t w = 0; + for (uint16_t code : col.codes) { + std::memcpy(out + w, slots + size_t{code} * StridedDictionary::kStride, kCopy); + w += lens[code]; + } + return w; +} + +#if defined(__ARM_FEATURE_SVE) +// As above, storing exactly the token's length. One predicate serves both sides: +// the load cannot read past the slot and the store writes no byte it does not own, +// so there is no over-copy and no width dispatch. +size_t DecompressIntoStridedExact(const StridedDictionary& dict, const Column& col, + uint8_t* out) { + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + size_t w = 0; + for (uint16_t code : col.codes) { + const uint8_t* src = slots + size_t{code} * StridedDictionary::kStride; + const uint32_t len = lens[code]; + svbool_t pg = svwhilelt_b8_u32(0u, len); + svst1_u8(pg, out + w, svld1_u8(pg, src)); + w += len; + } + return w; +} +#endif + +// Shared body of DecompressInto over the stored dictionary, for the streams too +// short to earn the strided view. template size_t DecompressIntoFixed(const Column& col, uint8_t* out) { const CompactDictionary& dict = col.dict; @@ -755,9 +827,20 @@ size_t DecompressIntoFixed(const Column& col, uint8_t* out) { size_t DecompressInto(const Column& col, uint8_t* out) { const size_t maxlen = col.dict.max_token_len; - if (maxlen <= 4) return DecompressIntoFixed<4>(col, out); - if (maxlen <= 8) return DecompressIntoFixed<8>(col, out); - return DecompressIntoFixed(col, out); + if (!StridedViewWorthBuilding(col.codes.size(), col.dict.num_tokens())) { + if (maxlen <= 4) return DecompressIntoFixed<4>(col, out); + if (maxlen <= 8) return DecompressIntoFixed<8>(col, out); + return DecompressIntoFixed(col, out); + } + StridedDictionary view; + view.Build(col.dict); +#if defined(__ARM_FEATURE_SVE) + return DecompressIntoStridedExact(view, col, out); +#else + if (maxlen <= 4) return DecompressIntoStrided<4>(view, col, out); + if (maxlen <= 8) return DecompressIntoStrided<8>(view, col, out); + return DecompressIntoStrided(view, col, out); +#endif } std::vector PackValues(const uint32_t* vals, size_t n, size_t bits) { @@ -776,21 +859,41 @@ std::vector PackValues(const uint32_t* vals, size_t n, size_t bits) { namespace { -// The gather-copy writes a fixed width per token so the copy length is a compile -// time constant, but that width only has to cover the longest token this -// dictionary actually holds -- not kMaxTokenSize. On corpora whose tokens are -// short the difference dominates decode: c_address averages 1.99 bytes per token, -// so a 16-byte copy moves 8x the bytes it needs to. +// What this loop actually waits on, measured by ablation across 30 corpora rather +// than inferred: the random dictionary read. Deleting the gather while keeping the +// unpack and the store nearly doubles throughput (+83%). Deleting the store while +// keeping the gather makes the loop SLOWER (-14%, on every corpus). So the store is +// not the constraint, and storing fewer bytes is not the lever. +// +// This corrects an earlier reading of the same code. The evidence then was that +// (over-copy factor) x (decode MiB/s) came out constant across corpora, which was +// taken to mean a fixed store-bandwidth ceiling. That product is equally constant +// when the loop is bound by tokens retired, because over-copy is +// kCopy / mean_token_len while throughput is mean_token_len x tokens_per_second -- +// the two models are indistinguishable from that measurement, and the ablation +// picks the other one. // -// Measured, this loop is store-bandwidth-bound. Across five unrelated corpora -// (over-copy factor) x (decode MiB/s) came out constant at ~10.6 GiB/s of store -// traffic, and the corpora with the highest over-copy decode slowest. Narrowing -// the width is therefore worth close to the bytes it saves. +// Two experiments were rejected under the old reading. Both really did lose, so +// they are recorded here, but the reason was misattributed: // -// The width is chosen once per stream from the dictionary, so there is no -// per-token branch: a predicate on token length would be nearly free on corpora -// where it always goes one way and expensive on the ones that split (urls sit at -// 41% short, the worst possible mix). +// - Unpacking codes a block at a time and prefetching before the gather: -22% +// with the offsets prefetched, -41% with the payload prefetched, worst case +// -65%, on all corpora. Aimed at the right cost, but a prefetch cannot help a +// stream of unpredictable indices arriving one code ahead of its use; it only +// adds a pass and the traffic of prefetches that arrive too late to hide +// anything. +// - A 12-byte copy as two stores: -4 to -6% wherever it applied. One wide store +// beats two narrow ones, which is a store-issue effect and holds regardless of +// what the loop is bound by. +// +// A third was measured and never built: choosing the copy width per block of 32 +// codes rather than per stream. Worth a median 1.00x of store traffic, because +// nearly every block of 32 contains at least one 16-byte token. +// +// The fix is to change what the gather reads. StridedDictionary gives each token a +// fixed slot and puts its length in a dense byte array, so one random line serves a +// token instead of two -- the layout FSST's decoder has always had, and which +// OnPair gave up when it lifted the length cap. See the kernels below. template size_t DecompressPackedFixed(const CompactDictionary& dict, const uint8_t* packed, size_t ncodes, size_t bits, uint8_t* out) { @@ -812,15 +915,6 @@ size_t DecompressPackedFixed(const CompactDictionary& dict, const uint8_t* packe // As above but with the code width a compile-time constant, so the mask folds to a // literal and `bitpos += kBits` strength-reduces. Dispatched once per stream, the // same way the copy width is. -// -// Tried and rejected here, so it is not re-attempted: unpacking codes a block at a -// time before gathering, to break the `w += len` store-address dependency and to -// prefetch the token bytes. It lost 22% with the offsets prefetched and 41% with -// dict.bytes prefetched (worst case -65%), across all 20 corpora. The premise was -// wrong -- this loop is store-bound, not latency-bound, which is the same thing the -// copy-width measurement showed. Breaking a dependency chain buys nothing against a -// store-bandwidth limit, and the extra pass plus 64 prefetches per block only add -// traffic. template size_t DecompressPackedFixedBits(const CompactDictionary& dict, const uint8_t* packed, size_t ncodes, uint8_t* out) { @@ -835,7 +929,8 @@ size_t DecompressPackedFixedBits(const CompactDictionary& dict, const uint8_t* p bitpos += kBits; // offsets[code] and offsets[code + 1] are adjacent u32s, so one 8-byte load // yields the token's start and end together. token_ptr/token_len would issue - // two loads for what is almost always a single cache line. + // two loads for what is almost always a single cache line. The payload still + // lives elsewhere, which is the second random line the strided view removes. uint64_t pair; std::memcpy(&pair, offsets_raw + size_t{code} * sizeof(uint32_t), sizeof(pair)); const uint32_t start = static_cast(pair); @@ -846,44 +941,158 @@ size_t DecompressPackedFixedBits(const CompactDictionary& dict, const uint8_t* p return w; } +// The same loop against the strided view: one random line per token, and the length +// read from a dense byte array small enough to stay resident. The bit-unpack +// prologue is unchanged, so this differs from the kernel above in the gather alone. +template +size_t DecompressStridedBits(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, uint8_t* out) { + constexpr uint32_t kMask = (kBits >= 32) ? 0xFFFFFFFFu : ((uint32_t{1} << kBits) - 1); + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + size_t bitpos = 0, w = 0; + for (size_t i = 0; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, packed + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & kMask; + bitpos += kBits; + std::memcpy(out + w, slots + size_t{code} * StridedDictionary::kStride, kCopy); + w += lens[code]; + } + return w; +} + +#if defined(__ARM_FEATURE_SVE) +// And with a predicated store, which is where most of the remaining gain is. One +// predicate covers the load and the store, so the loop reads only the token's own +// bytes and writes only the bytes it owns: the fixed over-copy is gone, and with it +// the reason to dispatch on max_token_len at all. +template +size_t DecompressStridedExactBits(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, uint8_t* out) { + constexpr uint32_t kMask = (kBits >= 32) ? 0xFFFFFFFFu : ((uint32_t{1} << kBits) - 1); + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + size_t bitpos = 0, w = 0; + for (size_t i = 0; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, packed + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & kMask; + bitpos += kBits; + const uint8_t* src = slots + size_t{code} * StridedDictionary::kStride; + const uint32_t len = lens[code]; + // kStride <= the SVE minimum vector length of 16 bytes, so this predicate never + // needs more lanes than the hardware has. + svbool_t pg = svwhilelt_b8_u32(0u, len); + svst1_u8(pg, out + w, svld1_u8(pg, src)); + w += len; + } + return w; +} +#endif + +// Runtime code width, for the widths training cannot produce but the format does +// not forbid. Kept so no input is rejected; never on a measured path. +template +size_t DecompressStridedFixed(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out) { + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + const uint32_t mask = (bits >= 32) ? 0xFFFFFFFFu : ((1u << bits) - 1); + size_t bitpos = 0, w = 0; + for (size_t i = 0; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, packed + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & mask; + bitpos += bits; + std::memcpy(out + w, slots + size_t{code} * StridedDictionary::kStride, kCopy); + w += lens[code]; + } + return w; +} + // Resolve `bits` to a constant for the widths a trained dictionary can produce // (kMinDictBits..kMaxDictBits), falling back to the runtime-width loop otherwise so // no input is rejected. +#define ONPAIR_DISPATCH_BITS(bits, CALL, FALLBACK) \ + switch (bits) { \ + case 9: return CALL(9); \ + case 10: return CALL(10); \ + case 11: return CALL(11); \ + case 12: return CALL(12); \ + case 13: return CALL(13); \ + case 14: return CALL(14); \ + case 15: return CALL(15); \ + case 16: return CALL(16); \ + default: return FALLBACK; \ + } + template size_t DecompressPackedDispatchBits(const CompactDictionary& dict, const uint8_t* packed, size_t ncodes, size_t bits, uint8_t* out) { - switch (bits) { - case 9: return DecompressPackedFixedBits(dict, packed, ncodes, out); - case 10: return DecompressPackedFixedBits(dict, packed, ncodes, out); - case 11: return DecompressPackedFixedBits(dict, packed, ncodes, out); - case 12: return DecompressPackedFixedBits(dict, packed, ncodes, out); - case 13: return DecompressPackedFixedBits(dict, packed, ncodes, out); - case 14: return DecompressPackedFixedBits(dict, packed, ncodes, out); - case 15: return DecompressPackedFixedBits(dict, packed, ncodes, out); - case 16: return DecompressPackedFixedBits(dict, packed, ncodes, out); - default: return DecompressPackedFixed(dict, packed, ncodes, bits, out); - } +#define ONPAIR_BLOB(B) DecompressPackedFixedBits(dict, packed, ncodes, out) + ONPAIR_DISPATCH_BITS(bits, ONPAIR_BLOB, + DecompressPackedFixed(dict, packed, ncodes, bits, out)) +#undef ONPAIR_BLOB } -} // namespace +template +size_t DecompressStridedDispatchBits(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out) { +#define ONPAIR_STRIDED(B) DecompressStridedBits(dict, packed, ncodes, out) + ONPAIR_DISPATCH_BITS(bits, ONPAIR_STRIDED, + DecompressStridedFixed(dict, packed, ncodes, bits, out)) +#undef ONPAIR_STRIDED +} -size_t DecompressPacked(const CompactDictionary& dict, const uint8_t* packed, size_t ncodes, - size_t bits, uint8_t* out) { +// Decode through a view, choosing the exact-length store where the target has one. +size_t DecompressThroughView(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out) { +#if defined(__ARM_FEATURE_SVE) +#define ONPAIR_STRIDED_EXACT(B) DecompressStridedExactBits(dict, packed, ncodes, out) + ONPAIR_DISPATCH_BITS(bits, ONPAIR_STRIDED_EXACT, + DecompressStridedFixed(dict, packed, ncodes, bits, out)) +#undef ONPAIR_STRIDED_EXACT +#else // Read the width, do not scan for it: an O(tokens) scan here costs 1-3% on // dictionaries of 20-60k tokens, which is charged to decode for something a // stored format keeps in its header. See CompactDictionary::max_token_len. - const size_t maxlen = dict.max_token_len; - // Only widths a single store can carry. A 12-byte copy moves 25% fewer bytes - // than 16 but needs two stores, and measured that loses 4-6% on every corpus it - // applied to (c_mktsegment, c_phone, p_container) -- so this is not purely a - // bandwidth effect and one wide store beats two narrow ones. Narrowing to 8 is - // worth 25-28% on the corpora that allow it. // - // `out` needs kDecodePadding of slack either way, and dict.bytes is read-padded - // by kMaxTokenSize, so every width here is in bounds. - if (maxlen <= 4) return DecompressPackedDispatchBits<4>(dict, packed, ncodes, bits, out); - if (maxlen <= 8) return DecompressPackedDispatchBits<8>(dict, packed, ncodes, bits, out); - return DecompressPackedDispatchBits(dict, packed, ncodes, bits, out); + // Only widths a single store can carry, for the reason recorded above: a 12-byte + // copy moves 25% fewer bytes than 16 but needs two stores and lost 4-6% wherever + // it applied. Narrowing to 8 is worth 25-28% on the corpora that allow it. + // + // `out` needs kDecodePadding of slack either way, and a slot is zero-filled out + // to kStride, so every width here is in bounds and reads defined bytes. + const size_t maxlen = dict.max_token_len; + if (maxlen <= 4) return DecompressStridedDispatchBits<4>(dict, packed, ncodes, bits, out); + if (maxlen <= 8) return DecompressStridedDispatchBits<8>(dict, packed, ncodes, bits, out); + return DecompressStridedDispatchBits(dict, packed, ncodes, bits, out); +#endif +} + +} // namespace + +size_t DecompressPacked(const StridedDictionary& dict, const uint8_t* packed, size_t ncodes, + size_t bits, uint8_t* out) { + return DecompressThroughView(dict, packed, ncodes, bits, out); +} + +size_t DecompressPacked(const CompactDictionary& dict, const uint8_t* packed, size_t ncodes, + size_t bits, uint8_t* out) { + if (!StridedViewWorthBuilding(ncodes, dict.num_tokens())) { + // See DecompressThroughView for why the width is read rather than scanned for, + // and why only 4/8/16 are offered. + const size_t maxlen = dict.max_token_len; + if (maxlen <= 4) return DecompressPackedDispatchBits<4>(dict, packed, ncodes, bits, out); + if (maxlen <= 8) return DecompressPackedDispatchBits<8>(dict, packed, ncodes, bits, out); + return DecompressPackedDispatchBits(dict, packed, ncodes, bits, out); + } + StridedDictionary view; + view.Build(dict); + return DecompressThroughView(view, packed, ncodes, bits, out); } +#undef ONPAIR_DISPATCH_BITS + } // namespace parquet::onpair diff --git a/cpp/src/parquet/onpair/onpair.h b/cpp/src/parquet/onpair/onpair.h index a0783b27caa2..741dadde9fb0 100644 --- a/cpp/src/parquet/onpair/onpair.h +++ b/cpp/src/parquet/onpair/onpair.h @@ -127,6 +127,39 @@ struct CompactDictionary { size_t logical_bytes() const { return offsets.empty() ? 0 : offsets.back(); } }; +/// A decode-side view of the dictionary: one cache line per token instead of two. +/// +/// CompactDictionary makes the decoder read two independent random locations per +/// token -- a u32 offsets pair from one array, then the payload from a +/// variable-stride blob. That gather, not the store traffic, is what the decode +/// loop waits on. Giving every token a fixed 16-byte slot and its length a byte in +/// a dense side array collapses the pair into one line, which is the layout FSST's +/// decoder has always used (fixed-stride `symbol[]` plus `len[]`). +/// +/// This is deliberately NOT the stored form. A 16-byte slot plus a length byte is +/// ~17 bytes per token against ~12 for blob-plus-offsets, so serializing it would +/// add hundreds of KiB on a 65k-token dictionary and move every compression ratio. +/// It is built once per column from the stored form and thrown away, so it costs +/// footprint only for the duration of a decode and nothing at all on disk. +struct StridedDictionary { + /// One slot per token. Also the decoder's fixed over-read width, and a power of + /// two <= 64, so a 64-byte-aligned base puts every slot inside a single line. + static constexpr size_t kStride = kMaxTokenSize; + + std::vector storage; ///< backing bytes; `slots` is aligned into this + uint8_t* slots = nullptr; ///< kStride bytes per token, 64-byte aligned + std::vector lens; ///< parallel token lengths, one byte each + /// Same meaning and same conservative default as CompactDictionary's: the + /// portable kernel's fixed copy width comes from it, so it must never + /// understate the true maximum. + size_t max_token_len = kMaxTokenSize; + + size_t num_tokens() const { return lens.size(); } + + /// Populate from a stored dictionary. O(tokens), once per column. + void Build(const CompactDictionary& dict); +}; + /// A compressed string column. `codes` is the row-concatenated code stream; /// row k is codes[row_offsets[k] .. row_offsets[k+1]]. struct Column { @@ -179,7 +212,18 @@ std::vector PackValues(const uint32_t* vals, size_t n, size_t bits); /// Decode a bit-packed code stream: read `bits` per code and gather-copy the /// token. `packed` needs >=4 trailing pad bytes; `out` >= DecodedLen + padding. +/// +/// Builds a StridedDictionary internally and decodes through it, except when the +/// stream is short enough that the O(tokens) build outweighs what it saves, in +/// which case the blob-and-offsets kernel runs directly. Decode a series of pages +/// against one dictionary through the overload below instead, so the build is paid +/// once rather than per page. size_t DecompressPacked(const CompactDictionary& dict, const uint8_t* packed, size_t ncodes, size_t bits, uint8_t* out); +/// Same, against a view built once by the caller. Output is byte-identical to the +/// CompactDictionary overload. +size_t DecompressPacked(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, size_t bits, uint8_t* out); + } // namespace parquet::onpair diff --git a/cpp/src/parquet/onpair/verify_roundtrip.cc b/cpp/src/parquet/onpair/verify_roundtrip.cc index 621e9fd60ffe..1a51c8994c1c 100644 --- a/cpp/src/parquet/onpair/verify_roundtrip.cc +++ b/cpp/src/parquet/onpair/verify_roundtrip.cc @@ -30,6 +30,7 @@ // cpp/src/parquet/onpair/onpair.cc cpp/src/parquet/onpair/verify_roundtrip.cc -o /tmp/verify // Run: /tmp/verify bench-fsst-onpair/corpora/tpch_l_shipmode.txt [more files...] +#include #include #include #include @@ -104,6 +105,68 @@ void Samples(const Corpus& c, const uint8_t* decoded) { } } +// Independent scalar reference decode: exact-length copies straight out of the +// stored dictionary, deliberately the dumbest loop that can be written. It shares +// no code with any shipped kernel, which is the point -- the kernels are checked +// against it rather than against each other. +std::vector ReferenceDecode(const op::CompactDictionary& dict, + const std::vector& codes, size_t n) { + std::vector out; + for (size_t i = 0; i < n; ++i) { + const uint8_t* p = dict.token_ptr(codes[i]); + out.insert(out.end(), p, p + dict.token_len(codes[i])); + } + return out; +} + +// Decode is served by several kernels chosen by target features and by stream +// length, and they must be interchangeable to the byte. Checked here: +// +// whole-column DecompressInto, unpacked u16 codes +// packed DecompressPacked, building its own strided view +// packed, prebuilt DecompressPacked against a view the caller built +// packed, short a prefix short enough to trip the guard that skips the view +// and decodes straight out of the stored dictionary +// +// The last one matters because nothing else reaches that path: real streams carry +// far more codes than tokens, so the fallback would otherwise never run here. +bool VerifyDecodePaths(const op::Column& col, const char* tag) { + const size_t ncodes = col.codes.size(); + const size_t ntokens = col.dict.num_tokens(); + size_t bits = 1; + while ((size_t{1} << bits) < ntokens) ++bits; + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), bits); + op::StridedDictionary view; + view.Build(col.dict); + + std::vector buf(op::DecodedLen(col) + op::kDecodePadding, 0); + size_t bad = 0; + auto agrees = [&](const char* what, const std::vector& want, size_t got) { + if (got == want.size() && std::memcmp(buf.data(), want.data(), want.size()) == 0) return; + std::printf(" %s: disagrees with the scalar reference (%zu vs %zu bytes)\n", what, + got, want.size()); + ++bad; + }; + + const std::vector want_all = ReferenceDecode(col.dict, col.codes, ncodes); + agrees("whole-column", want_all, op::DecompressInto(col, buf.data())); + agrees("packed", want_all, + op::DecompressPacked(col.dict, packed.data(), ncodes, bits, buf.data())); + agrees("packed, prebuilt view", want_all, + op::DecompressPacked(view, packed.data(), ncodes, bits, buf.data())); + + const size_t nshort = std::min(ncodes, ntokens == 0 ? 0 : ntokens - 1); + if (nshort != 0) { + agrees("packed, short stream", ReferenceDecode(col.dict, col.codes, nshort), + op::DecompressPacked(col.dict, packed.data(), nshort, bits, buf.data())); + } + + std::printf(" %-15s: 4 decode paths agree (%zu tokens, %zub codes, max token %zu) %s\n", + tag, ntokens, bits, col.dict.max_token_len, bad == 0 ? "[OK]" : "[FAIL]"); + return bad == 0; +} + // OnPair (no dedup): compress then whole-column decode. bool VerifyOnPair(const Corpus& c, uint8_t bits, double threshold, bool show_samples) { op::Config cfg{bits, threshold, 42}; @@ -115,8 +178,13 @@ bool VerifyOnPair(const Corpus& c, uint8_t bits, double threshold, bool show_sam std::printf(" OnPair%-2u : %zu/%zu rows exact %s\n", bits, c.rows() - bad, c.rows(), bad == 0 ? "[OK]" : "[FAIL]"); if (bad != 0) std::printf(" first mismatching row: %ld\n", bad_at); + // Every width gets the four-path check, not just 16: each width instantiates a + // different unpack kernel, so agreement at one width says nothing about another. + char tag[32]; + std::snprintf(tag, sizeof(tag), "OnPair%u paths", static_cast(bits)); + bool paths = VerifyDecodePaths(col, tag); if (show_samples) Samples(c, out.data()); - return bad == 0; + return paths && bad == 0; } // The distinct-value set a dedup cascade OnPairs, plus the per-row ids into it. @@ -174,7 +242,10 @@ bool VerifyOnPairDedup(const Corpus& c, const Distinct& d, uint8_t bits, double c.rows() - bad, c.rows(), d.count(), bad == 0 ? "[OK]" : "[FAIL]"); if (bad != 0) std::printf(" first mismatching row: %ld\n", bad_at); if (show_samples) Samples(c, out.data()); - return bad == 0; + // A distinct-value dictionary is far smaller than a whole column's, so its codes + // pack into a narrower width than OnPair ever picks -- this is where the low end + // of the width dispatch gets exercised. + return VerifyDecodePaths(col, " ↳ paths") && bad == 0; } } // namespace From 9363e1d33cf8fbafea29a56f0f8e162e50a23a10 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Tue, 4 Aug 2026 11:03:18 +0000 Subject: [PATCH 23/24] onpair: decode a group of codes per iteration instead of one At a fixed code width the (byte offset, intra-byte shift) pair a packed code is read with repeats with a period of 8 / gcd(width, 8) codes. Unrolling the decode loop by a whole multiple of that period makes every offset and every shift a compile-time constant and advances the stream cursor once per group rather than once per code. The group size is a request that is rounded up to a whole period, so it is a floor rather than the group emitted. That is what lets one setting be constant-addressed at every width including 16, where the period is a single code and unrolling by the period alone would be a no-op -- which is exactly the high-cardinality half of the corpus set. Measured with the builds alternating many times over, pinned, order rotated and reversed between repetitions, every comparison a paired difference taken inside one repetition: +4.6% on the corpus-set median with the predicated store and +20.7% on the portable build, +34% on the two 14-bit columns where it removes a defect in the old loop rather than raising a ceiling. Nothing about the compressed form is touched, so no ratio moves. The verifier gains a sweep over sixteen consecutive stream lengths per column, which covers every remainder any group size up to sixteen can leave after its last whole group. A column whose code count is a whole multiple of the group never reaches the remainder path, so a whole-column check alone cannot see a fault there: with the remainder loop deliberately made to skip its first code, 43 of the 120 checkpoints are caught by the length sweep and by nothing else. An empty corpus is now a failure rather than a vacuous pass. --- cpp/src/parquet/onpair/onpair.cc | 208 +++++++++++++++++++++ cpp/src/parquet/onpair/verify_roundtrip.cc | 39 +++- 2 files changed, 241 insertions(+), 6 deletions(-) diff --git a/cpp/src/parquet/onpair/onpair.cc b/cpp/src/parquet/onpair/onpair.cc index af5882654aaf..bdb0c1201ac0 100644 --- a/cpp/src/parquet/onpair/onpair.cc +++ b/cpp/src/parquet/onpair/onpair.cc @@ -21,6 +21,7 @@ #include #include #include +#include // The decode loop stores exactly one token's length per iteration when a // predicated store is available, which removes the fixed over-copy entirely. This @@ -991,6 +992,204 @@ size_t DecompressStridedExactBits(const StridedDictionary& dict, const uint8_t* } #endif +// Which decode form to build. Set on the command line to measure one against +// another with nothing else in the build changed. +// +// 0 the per-code loops above, no unroll at all +// 1 groups of one phase period (below); at a 16-bit width the period is a single +// code, so this leaves those columns on the per-code loop +// 2 groups of at least ONPAIR_GROUP_CODES codes at every width, 16 included +#ifndef ONPAIR_GROUP_UNROLL +#define ONPAIR_GROUP_UNROLL 2 +#endif + +// 0 asks for the phase period itself, the smallest group that makes every offset +// and shift constant. A nonzero value asks for at least that many codes, rounded up +// to a whole period, so the request is a floor rather than the group emitted: four +// emits eight codes at the widths whose period is eight and four at 14 and 16, where +// the request is taken literally. +// +// Four is measured, not chosen for its arithmetic. Sweeping the request over 2, 4, 8 +// and 16 only moves the widths where the request is not rounded up, and a 16-bit width +// is the only one of those where the group size is genuinely free. What happens there +// depends on the build, which is the more interesting result of the sweep. With the +// predicated store the answer is a step: two codes gains nothing at all and four gains +// the lot. On the portable build it is a slope, two already gaining about half of what +// eight gains, close to the (1 - 1/n) curve a loop whose group saving is per-code +// cursor arithmetic would follow. So the group form is removing scalar work in one +// build and covering dictionary-read latency in the other, and neither reading +// generalizes to the other. Four is the smallest request that captures the bulk of the +// gain on both. +// +// Above four the sweep stops choosing. Four and eight differ by well under a per cent +// at the only widths where they differ at all, and the two builds disagree on which +// way: with the predicated store four is ahead on 12 of the 15 16-bit columns, and on +// the portable build eight is ahead, in both portable sweeps. This is a tie broken on +// code size and on the shorter remainder a smaller group leaves after its last whole +// group, which matters on a short page and which the corpora here are all too long to +// show. +#if ONPAIR_GROUP_UNROLL >= 2 +#ifndef ONPAIR_GROUP_CODES +#define ONPAIR_GROUP_CODES 4 +#endif +constexpr size_t kGroupCodesRequest = ONPAIR_GROUP_CODES; +#else +constexpr size_t kGroupCodesRequest = 0; +#endif + +// Group unroll on the bit cursor's phase period. +// +// At a fixed code width the (byte offset, intra-byte shift) pair a code is read +// with repeats with period 8 / gcd(kBits, 8) codes: 8 at the widths coprime with 8, +// 4 at 10 and 14, 2 at 12, 1 at 16. Unrolling by that period turns every offset and +// every shift into a compile-time constant and advances the stream pointer once per +// group instead of once per code, so the cursor arithmetic the loops above do per +// code disappears. +// +// Any whole multiple of the period has the same property, so asking for a fixed +// number of codes and rounding up to a whole period is constant-addressed at every +// width. That matters because the period is one code at a 16-bit width -- there is +// no phase to fold -- and a trained 16-bit code space lands there on half the +// corpora, so unrolling by the period alone leaves exactly the largest-dictionary +// columns on the per-code loop. Asking for a group instead unrolls those columns on +// the strength of having several independent gathers in flight rather than on folded +// addressing. +// +// Sorting the measured gain by code width rather than by period separates the two +// effects. The 16-bit columns gain with no phase to fold at all, which is the part +// that is not addressing; the 15-bit columns, the worst phase case, gain about three +// times as much at the same instruction count per code, which is the part that is. +// Both terms are real and neither accounts for the whole. Width and dictionary size +// move together, though, so this does not fully separate cheaper addressing from a +// dictionary that misses L1. +// +// Nothing else changes: the same single fused pass, the same gather, the same +// store, no staging buffer, no prefetch, no added memory traffic. That is what +// separates this from the block-wise unpack recorded above as a loss, which paid an +// extra pass and 64 prefetches per block for the same codes. +// +// The reason to expect anything here is that the ablation ladder above does not +// actually isolate the gather. Deleting the gather deletes its two loads and their +// address arithmetic as well, so that rung bounds the gather plus its instructions, +// not the gather alone -- the same confound as the store-bandwidth reading it +// replaced. This change removes instructions while leaving the gather byte for +// byte identical, so it separates the two where neither ablation could. +// +// A group's last code is read by a 4-byte load at a constant offset, which runs at +// most 3 bytes past the group at every width and group size used here. PackValues +// carries 4 zero-filled tail bytes and the per-code loops read just as far, so the +// precondition on `packed` is unchanged. +template +struct PackedCodeGroup { + static constexpr size_t Gcd(size_t a, size_t b) { return b == 0 ? a : Gcd(b, a % b); } + static constexpr size_t kPeriod = 8 / Gcd(kBits, 8); + // A group has to consume a whole number of bytes so the next group starts at bit + // offset zero again. That holds for any multiple of the period, so round up. + static constexpr size_t kCodes = + kRequest == 0 ? kPeriod : ((kRequest + kPeriod - 1) / kPeriod) * kPeriod; + static constexpr size_t kBytes = kBits * kCodes / 8; + static_assert(kBytes * 8 == kBits * kCodes, "a group must be a whole number of bytes"); +}; + +template +inline uint32_t GroupCodeAt(const uint8_t* p) { + constexpr size_t kOff = (J * kBits) / 8; + constexpr size_t kShift = (J * kBits) % 8; + static_assert(kBits >= 1 && kBits <= 25, "the mask below would overflow"); + static_assert(kShift + kBits <= 32, "one 4-byte load must cover the code"); + constexpr uint32_t kMask = (uint32_t{1} << kBits) - 1; + uint32_t word; + std::memcpy(&word, p + kOff, sizeof(word)); + return (word >> kShift) & kMask; +} + +// One call per code in the group, in stream order. A fold over the comma operator +// is sequenced left to right, which the callers rely on: the write cursor advances +// by each token's own length in turn. +template +inline void ForEachCodeInGroup(const uint8_t* p, Emit emit, std::index_sequence) { + (emit(GroupCodeAt(p)), ...); +} + +// The strided portable loop, group-unrolled. +template +size_t DecompressStridedGroupBits(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, uint8_t* out) { + using Group = PackedCodeGroup; + // With the phase period asked for and a 16-bit width, the group is a single code: + // no phase to fold, and the form would be the per-code loop with the shift known + // to be zero. It measured indistinguishable from the plain loop, so at a group of + // one this defers rather than emitting the same code a second time. Any request of + // two or more never lands here. + if constexpr (Group::kCodes <= 1) { + return DecompressStridedBits(dict, packed, ncodes, out); + } + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + const uint8_t* p = packed; + size_t w = 0; + auto emit = [&](uint32_t code) { + std::memcpy(out + w, slots + size_t{code} * StridedDictionary::kStride, kCopy); + w += lens[code]; + }; + for (size_t g = 0, ngroups = ncodes / Group::kCodes; g < ngroups; ++g) { + ForEachCodeInGroup(p, emit, std::make_index_sequence{}); + p += Group::kBytes; + } + // A whole number of groups consumes a whole number of bytes, so the cursor is + // byte aligned here and the tail starts its own bit offset from zero. + constexpr uint32_t kMask = (uint32_t{1} << kBits) - 1; + size_t bitpos = 0; + for (size_t i = (ncodes / Group::kCodes) * Group::kCodes; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, p + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & kMask; + bitpos += kBits; + std::memcpy(out + w, slots + size_t{code} * StridedDictionary::kStride, kCopy); + w += lens[code]; + } + return w; +} + +#if defined(__ARM_FEATURE_SVE) +// And the predicated-store loop, group-unrolled. Same two changes composed: one +// random line per token, exactly the token's bytes stored, constant addressing. +template +size_t DecompressStridedGroupExactBits(const StridedDictionary& dict, const uint8_t* packed, + size_t ncodes, uint8_t* out) { + using Group = PackedCodeGroup; + // See above: at a group of one there is no phase to fold, and that form measured + // slower than the plain loop. + if constexpr (Group::kCodes <= 1) { + return DecompressStridedExactBits(dict, packed, ncodes, out); + } + const uint8_t* slots = dict.slots; + const uint8_t* lens = dict.lens.data(); + const uint8_t* p = packed; + size_t w = 0; + auto emit = [&](uint32_t code) { + const uint32_t len = lens[code]; + svbool_t pg = svwhilelt_b8_u32(0u, len); + svst1_u8(pg, out + w, svld1_u8(pg, slots + size_t{code} * StridedDictionary::kStride)); + w += len; + }; + for (size_t g = 0, ngroups = ncodes / Group::kCodes; g < ngroups; ++g) { + ForEachCodeInGroup(p, emit, std::make_index_sequence{}); + p += Group::kBytes; + } + constexpr uint32_t kMask = (uint32_t{1} << kBits) - 1; + size_t bitpos = 0; + for (size_t i = (ncodes / Group::kCodes) * Group::kCodes; i < ncodes; ++i) { + uint32_t word; + std::memcpy(&word, p + (bitpos >> 3), 4); + uint32_t code = (word >> (bitpos & 7)) & kMask; + bitpos += kBits; + emit(code); + } + return w; +} +#endif + // Runtime code width, for the widths training cannot produce but the format does // not forbid. Kept so no input is rejected; never on a measured path. template @@ -1039,7 +1238,11 @@ size_t DecompressPackedDispatchBits(const CompactDictionary& dict, const uint8_t template size_t DecompressStridedDispatchBits(const StridedDictionary& dict, const uint8_t* packed, size_t ncodes, size_t bits, uint8_t* out) { +#if ONPAIR_GROUP_UNROLL +#define ONPAIR_STRIDED(B) DecompressStridedGroupBits(dict, packed, ncodes, out) +#else #define ONPAIR_STRIDED(B) DecompressStridedBits(dict, packed, ncodes, out) +#endif ONPAIR_DISPATCH_BITS(bits, ONPAIR_STRIDED, DecompressStridedFixed(dict, packed, ncodes, bits, out)) #undef ONPAIR_STRIDED @@ -1049,7 +1252,12 @@ size_t DecompressStridedDispatchBits(const StridedDictionary& dict, const uint8_ size_t DecompressThroughView(const StridedDictionary& dict, const uint8_t* packed, size_t ncodes, size_t bits, uint8_t* out) { #if defined(__ARM_FEATURE_SVE) +#if ONPAIR_GROUP_UNROLL +#define ONPAIR_STRIDED_EXACT(B) \ + DecompressStridedGroupExactBits(dict, packed, ncodes, out) +#else #define ONPAIR_STRIDED_EXACT(B) DecompressStridedExactBits(dict, packed, ncodes, out) +#endif ONPAIR_DISPATCH_BITS(bits, ONPAIR_STRIDED_EXACT, DecompressStridedFixed(dict, packed, ncodes, bits, out)) #undef ONPAIR_STRIDED_EXACT diff --git a/cpp/src/parquet/onpair/verify_roundtrip.cc b/cpp/src/parquet/onpair/verify_roundtrip.cc index 1a51c8994c1c..b9bcfe7efa9c 100644 --- a/cpp/src/parquet/onpair/verify_roundtrip.cc +++ b/cpp/src/parquet/onpair/verify_roundtrip.cc @@ -127,9 +127,18 @@ std::vector ReferenceDecode(const op::CompactDictionary& dict, // packed, prebuilt DecompressPacked against a view the caller built // packed, short a prefix short enough to trip the guard that skips the view // and decodes straight out of the stored dictionary +// packed, tails sixteen consecutive lengths, covering every remainder // -// The last one matters because nothing else reaches that path: real streams carry +// The short case matters because nothing else reaches that path: real streams carry // far more codes than tokens, so the fallback would otherwise never run here. +// +// The tail sweep matters for a different reason. The packed kernels decode a fixed +// group of codes per iteration and hand whatever is left to the per-code loop, so a +// length one code short of a whole group is where a mistake in that handover shows +// up. The short case above cannot be relied on for that: its length is fixed by the +// dictionary size, so which remainder it lands on is an accident. Sweeping sixteen +// consecutive lengths covers every remainder any group size up to sixteen can leave, +// whatever the group size is compiled to be. bool VerifyDecodePaths(const op::Column& col, const char* tag) { const size_t ncodes = col.codes.size(); const size_t ntokens = col.dict.num_tokens(); @@ -142,12 +151,16 @@ bool VerifyDecodePaths(const op::Column& col, const char* tag) { std::vector buf(op::DecodedLen(col) + op::kDecodePadding, 0); size_t bad = 0; - auto agrees = [&](const char* what, const std::vector& want, size_t got) { - if (got == want.size() && std::memcmp(buf.data(), want.data(), want.size()) == 0) return; + auto agrees_bytes = [&](const char* what, const uint8_t* want, size_t want_len, + size_t got) { + if (got == want_len && std::memcmp(buf.data(), want, want_len) == 0) return; std::printf(" %s: disagrees with the scalar reference (%zu vs %zu bytes)\n", what, - got, want.size()); + got, want_len); ++bad; }; + auto agrees = [&](const char* what, const std::vector& want, size_t got) { + agrees_bytes(what, want.data(), want.size(), got); + }; const std::vector want_all = ReferenceDecode(col.dict, col.codes, ncodes); agrees("whole-column", want_all, op::DecompressInto(col, buf.data())); @@ -162,8 +175,22 @@ bool VerifyDecodePaths(const op::Column& col, const char* tag) { op::DecompressPacked(col.dict, packed.data(), nshort, bits, buf.data())); } - std::printf(" %-15s: 4 decode paths agree (%zu tokens, %zub codes, max token %zu) %s\n", - tag, ntokens, bits, col.dict.max_token_len, bad == 0 ? "[OK]" : "[FAIL]"); + // Decoding a prefix of the codes yields a prefix of the whole-column bytes, so the + // expected output is want_all truncated -- no need to decode a reference per tail. + // The prebuilt view is used so the guard that falls back to the stored dictionary + // on a short stream cannot skip the kernel under test. + size_t tails = 0, want_len = want_all.size(); + for (size_t k = 1; k <= 16 && k < ncodes; ++k) { + want_len -= col.dict.token_len(col.codes[ncodes - k]); + agrees_bytes("packed, tail", want_all.data(), want_len, + op::DecompressPacked(view, packed.data(), ncodes - k, bits, buf.data())); + ++tails; + } + + std::printf(" %-15s: %zu decode paths agree (%zu tokens, %zub codes, " + "max token %zu) %s\n", + tag, 4 + tails, ntokens, bits, col.dict.max_token_len, + bad == 0 ? "[OK]" : "[FAIL]"); return bad == 0; } From 6193303e92bfbcc88eb809d6c83c614f9841d101 Mon Sep 17 00:00:00 2001 From: Prateek Gaur Date: Mon, 3 Aug 2026 08:19:02 +0000 Subject: [PATCH 24/24] Train FSST's symbol table at a 16-bit code space and compare it to OnPair The reference FSST trainer cannot be widened in place. Its pair counter is a dense square over the code space, which is a few hundred kilobytes at nine bits and tens of gigabytes at sixteen, and its symbol type packs the bytes into a single machine word, capping a symbol at eight bytes. Both are replaced here, the counter by an open-addressed map over occupied pairs and the symbol by a fixed byte array with an explicit length, so the same iterative local search can fill a sixteen-bit table with symbols as long as OnPair's. The trained table is handed to OnPair's own encoder through a new train-free entry point, so the parsing pass and the decode kernel are shared and only the table construction differs. That makes the ratio and decode columns attributable to the table alone. The round-trip verifier now gates rather than reports, returning non-zero on any failure, and it decodes every FSST16 configuration twice, once through the whole-column path and once through the bit-packed path at the width the trained table needs, because a table of a few hundred tokens exercises a much narrower packed loop than OnPair ever reaches. --- cpp/src/parquet/onpair/fsst16.cc | 478 ++++++++++++++++++ cpp/src/parquet/onpair/fsst16.h | 144 ++++++ .../parquet/onpair/fsst_onpair_benchmark.cc | 119 ++++- cpp/src/parquet/onpair/onpair.cc | 20 + cpp/src/parquet/onpair/onpair.h | 14 + cpp/src/parquet/onpair/verify_roundtrip.cc | 83 ++- 6 files changed, 839 insertions(+), 19 deletions(-) create mode 100644 cpp/src/parquet/onpair/fsst16.cc create mode 100644 cpp/src/parquet/onpair/fsst16.h diff --git a/cpp/src/parquet/onpair/fsst16.cc b/cpp/src/parquet/onpair/fsst16.cc new file mode 100644 index 000000000000..6c348ce3d514 --- /dev/null +++ b/cpp/src/parquet/onpair/fsst16.cc @@ -0,0 +1,478 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "parquet/onpair/fsst16.h" + +#include +#include +#include + +namespace parquet::fsst16 { +namespace { + +// The first 256 codes are the literal bytes; learned symbols start here. +constexpr size_t kCodeBase = 256; +// Longest run of one row the sample generator takes in a single bite. The +// reference's constant. +constexpr size_t kSampleLine = 512; + +// The reference's integer hash, used for sample selection and round skipping so +// that both pick the same rows it would. +inline uint64_t FsstHash(uint64_t w) { + const uint64_t p = w * 2971215073ull; + return p ^ (p >> 15); +} + +// Byte-range hash for the symbol indexes below: eight bytes a step with a tail, +// then an avalanche. +inline uint64_t HashBytes(const uint8_t* p, size_t len) { + uint64_t h = 0x9E3779B97F4A7C15ull ^ (static_cast(len) * 0xff51afd7ed558ccdull); + size_t i = 0; + for (; i + 8 <= len; i += 8) { + uint64_t w; + std::memcpy(&w, p + i, 8); + h = (h ^ w) * 0x100000001b3ull; + } + if (i < len) { + uint64_t w = 0; + std::memcpy(&w, p + i, len - i); + h = (h ^ w) * 0x100000001b3ull; + } + h ^= h >> 29; + h *= 0xbf58476d1ce4e5b9ull; + h ^= h >> 32; + return h; +} + +struct Sym { + uint8_t b[kMaxSymbolLen]; + uint8_t len; +}; + +// Concatenation, truncated at the length cap exactly as the reference truncates +// at eight bytes rather than rejecting the pair. +inline Sym Concat(const Sym& a, const Sym& b, size_t cap) { + Sym s; + size_t n = std::min(static_cast(a.len) + b.len, cap); + std::memcpy(s.b, a.b, a.len); + if (n > a.len) std::memcpy(s.b + a.len, b.b, n - a.len); + s.len = static_cast(n); + return s; +} + +// Longest-prefix index over the symbol table. +// +// The reference indexes with a two-byte lookup array plus a three-byte-prefix +// hash, both of which lean on a symbol fitting in one machine word. A 16-byte +// symbol does not fit, so this keeps one open-addressed set per symbol length +// and probes them longest-first, skipping lengths the table has none of. It only +// ever runs over the training sample - tens of KiB, five times - so a few probes +// per position costs far less than constraining what a symbol may be. +class SymbolIndex { + public: + explicit SymbolIndex(const std::vector* syms) : syms_(syms), slots_(kSlots, 0) {} + + void Clear() { + std::fill(slots_.begin(), slots_.end(), 0u); + len_present_ = 0; + max_len_ = 1; + } + + /// False if a symbol with these exact bytes is already indexed. + bool Insert(const Sym& s, uint32_t code) { + size_t i = Probe(s.b, s.len); + if (slots_[i] != 0) return false; + slots_[i] = code + 1; + len_present_ |= uint32_t{1} << s.len; + if (s.len > max_len_) max_len_ = s.len; + return true; + } + + /// Code of the longest symbol that prefixes [p, p+n). Falls back to the + /// literal code for the first byte, which is always resident. + uint32_t Find(const uint8_t* p, size_t n) const { + size_t hi = std::min(max_len_, n); + for (size_t len = hi; len >= 2; --len) { + if (((len_present_ >> len) & 1u) == 0) continue; + size_t i = Probe(p, len); + if (slots_[i] != 0) return slots_[i] - 1; + } + return p[0]; + } + + private: + // Twice the 65536-symbol ceiling, so the set never passes half load and the + // probe chains stay short without any rehashing. + static constexpr size_t kSlots = size_t{1} << 17; + + size_t Probe(const uint8_t* p, size_t len) const { + size_t i = HashBytes(p, len) & (kSlots - 1); + while (slots_[i] != 0) { + const Sym& s = (*syms_)[slots_[i] - 1]; + if (s.len == len && std::memcmp(s.b, p, len) == 0) break; + i = (i + 1) & (kSlots - 1); + } + return i; + } + + const std::vector* syms_; + std::vector slots_; // code + 1, or 0 for empty + uint32_t len_present_ = 0; // bit L set when some symbol has length L + uint32_t max_len_ = 1; +}; + +// Adjacent-pair frequencies, sparse (V1). +// +// Keyed by the two codes packed into one word. Occupied slots are listed as they +// are first written so that clearing between rounds and walking the pairs for +// candidate generation both cost the number of distinct pairs rather than the +// table size. +class PairCounts { + public: + void Reset(size_t expected_pairs) { + size_t want = 1024; + const size_t ceiling = size_t{1} << 22; // grows past this only if a column needs it + while (want < 2 * expected_pairs && want < ceiling) want <<= 1; + if (want > slots_) { + Allocate(want); + } else { + Clear(); + } + } + + void Clear() { + for (size_t i : used_) { + keys_[i] = 0; + vals_[i] = 0; + } + used_.clear(); + } + + void Inc(uint32_t c1, uint32_t c2) { + if (2 * used_.size() >= slots_) Grow(); + const uint64_t key = (static_cast(c1) << 16 | c2) + 1; // 0 marks empty + size_t i = Slot(key); + if (keys_[i] == 0) { + keys_[i] = key; + used_.push_back(i); + } + ++vals_[i]; + } + + const std::vector& used() const { return used_; } + uint32_t left(size_t slot) const { return static_cast((keys_[slot] - 1) >> 16); } + uint32_t right(size_t slot) const { return static_cast((keys_[slot] - 1) & 0xFFFF); } + uint32_t count(size_t slot) const { return vals_[slot]; } + + private: + void Allocate(size_t want) { + slots_ = want; + keys_.assign(slots_, 0); + vals_.assign(slots_, 0); + used_.clear(); + } + + size_t Slot(uint64_t key) const { + size_t i = FsstHash(key) & (slots_ - 1); + while (keys_[i] != 0 && keys_[i] != key) i = (i + 1) & (slots_ - 1); + return i; + } + + // Kept at or below half load so probe chains stay short. + void Grow() { + std::vector old_keys = std::move(keys_); + std::vector old_vals = std::move(vals_); + std::vector old_used = std::move(used_); + Allocate(slots_ * 2); + for (size_t o : old_used) { + const size_t i = Slot(old_keys[o]); + keys_[i] = old_keys[o]; + vals_[i] = old_vals[o]; + used_.push_back(i); + } + } + + size_t slots_ = 0; + std::vector keys_; // packed pair + 1 + std::vector vals_; + std::vector used_; +}; + +struct Cand { + Sym sym; + uint64_t score; +}; + +// Candidate set for one round, deduplicating by symbol bytes and summing the +// scores of duplicates, as the reference's candidate set does. +class CandSet { + public: + void Reset(size_t expected) { + size_t want = 16; + while (want < 2 * expected + 16) want <<= 1; + slots_.assign(want, kEmpty); + cands_.clear(); + } + + void AddOrInc(const Sym& s, uint64_t score) { + const size_t mask = slots_.size() - 1; + size_t i = HashBytes(s.b, s.len) & mask; + while (slots_[i] != kEmpty) { + Cand& c = cands_[slots_[i]]; + if (c.sym.len == s.len && std::memcmp(c.sym.b, s.b, s.len) == 0) { + c.score += score; + return; + } + i = (i + 1) & mask; + } + slots_[i] = static_cast(cands_.size()); + cands_.push_back({s, score}); + } + + std::vector& cands() { return cands_; } + + private: + static constexpr uint32_t kEmpty = 0xFFFFFFFFu; + std::vector slots_; + std::vector cands_; +}; + +class Trainer { + public: + explicit Trainer(const Config& cfg) + : cfg_(cfg), + cap_(std::min(std::max(cfg.max_symbol_len, 2), kMaxSymbolLen)), + index_(&syms_) { + count1_.assign(cfg_.max_symbols, 0); + } + + Tokens Run(const uint8_t* data, const uint32_t* offsets, size_t n) { + BuildSample(data, offsets, n); + size_t sample_bytes = 0; + for (const auto& l : lines_) sample_bytes += l.second; + pairs_.Reset(2 * sample_bytes + 16); + + ResetTable(); + + int64_t best_gain = INT64_MIN; + std::vector best_syms = syms_; + std::vector best_count1 = count1_; + + // Five rounds at sample fractions 8, 38, 68, 98, 128; the last measures the + // table it inherits without proposing a new one. + for (size_t frac = 8;; frac += 30) { + std::fill(count1_.begin(), count1_.end(), 0u); + pairs_.Clear(); + const int64_t gain = CompressCount(frac); + if (gain >= best_gain) { + best_gain = gain; + best_syms = syms_; + best_count1 = count1_; + } + if (frac >= 128) break; + MakeTable(frac); + } + + // Rebuild the winning table from its own single-symbol counts, dropping the + // symbols the winning round never actually used. + syms_ = std::move(best_syms); + count1_ = std::move(best_count1); + pairs_.Clear(); + MakeTable(128); + + return Emit(); + } + + private: + // Sample selection, following the reference: take the whole column when it is + // under the target, otherwise fill the target with randomly chosen runs of + // randomly chosen rows. + void BuildSample(const uint8_t* data, const uint32_t* offsets, size_t n) { + const size_t total = n == 0 ? 0 : offsets[n]; + if (total <= cfg_.sample_target) { + for (size_t i = 0; i < n; ++i) { + if (offsets[i + 1] > offsets[i]) { + lines_.emplace_back(data + offsets[i], offsets[i + 1] - offsets[i]); + } + } + return; + } + + std::vector> spans; // (offset in buf, length) + buf_.reserve(cfg_.sample_target + kSampleLine); + uint64_t rnd = FsstHash(cfg_.seed); + while (buf_.size() < cfg_.sample_target) { + rnd = FsstHash(rnd); + size_t row = rnd % n; + while (offsets[row + 1] == offsets[row]) { + if (++row == n) row = 0; + } + const size_t len = offsets[row + 1] - offsets[row]; + const size_t chunks = 1 + (len - 1) / kSampleLine; + rnd = FsstHash(rnd); + const size_t chunk = kSampleLine * (rnd % chunks); + const size_t take = std::min(len - chunk, kSampleLine); + const uint8_t* src = data + offsets[row] + chunk; + spans.emplace_back(buf_.size(), take); + buf_.insert(buf_.end(), src, src + take); + } + // Resolved after the buffer stops growing, since inserting reallocates it. + for (const auto& sp : spans) lines_.emplace_back(buf_.data() + sp.first, sp.second); + } + + void ResetTable() { + syms_.resize(kCodeBase); + for (size_t i = 0; i < kCodeBase; ++i) { + syms_[i].len = 1; + syms_[i].b[0] = static_cast(i); + } + index_.Clear(); + } + + void AddSym(const Sym& s) { + if (s.len < 2) return; // V4: the literals are already resident + const uint32_t code = static_cast(syms_.size()); + syms_.push_back(s); + if (!index_.Insert(s, code)) syms_.pop_back(); + } + + // Round skipping: the reference's per-row draw in 1..128. + static size_t Rnd128(size_t i, size_t frac) { + return 1 + (FsstHash((i + 1) * frac) & 127); + } + + // Compress the sample with the current table, counting single symbols and + // adjacent pairs, and return the gain: bytes saved against storing the sample + // raw, given that every code costs two bytes. + int64_t CompressCount(size_t frac) { + int64_t gain = 0; + for (size_t i = 0; i < lines_.size(); ++i) { + if (frac < 128 && Rnd128(i, frac) > frac) continue; + const uint8_t* cur = lines_[i].first; + const uint8_t* end = cur + lines_[i].second; + if (cur >= end) continue; + const uint8_t* start = cur; + + uint32_t code1 = index_.Find(cur, end - cur); + cur += syms_[code1].len; + gain += static_cast(syms_[code1].len) - 2; + + for (;;) { + // Not extending this symbol is one option, so count it alone. + ++count1_[code1]; + // Taking just its first byte is the other, unless they are the same. + if (syms_[code1].len != 1) ++count1_[*start]; + + if (cur == end) break; + + start = cur; + const uint32_t code2 = index_.Find(cur, end - cur); + cur += syms_[code2].len; + gain += static_cast(cur - start) - 2; + + if (frac < 128) { // the last round proposes nothing, so counts no pairs + pairs_.Inc(code1, code2); + if (cur - start > 1) pairs_.Inc(code1, *start); + } + code1 = code2; + } + } + return gain; + } + + // Clear the table and refill it from the highest-scoring candidates. + void MakeTable(size_t frac) { + const uint64_t min_count = (5 * frac) / 128; + cands_.Reset(syms_.size() + pairs_.used().size()); + + // Every counted symbol is a candidate to keep. + for (size_t p1 = 0; p1 < syms_.size(); ++p1) { + const uint32_t c1 = count1_[p1]; + if (c1 == 0) continue; + const Sym& s1 = syms_[p1]; + // V4: promoting single bytes is the reference's way of holding its escape + // rate down. Kept so scores match, though a length-1 candidate is never + // admitted here. + const uint64_t cnt = (s1.len == 1 ? 8ull : 1ull) * c1; + if (cnt < min_count) continue; + cands_.AddOrInc(s1, cnt * s1.len); + } + + // Every counted pair is a candidate to merge (V2). The last round proposes + // nothing, matching the reference's refusal to grow symbols there. + if (frac < 128) { + for (size_t slot : pairs_.used()) { + const uint32_t cnt = pairs_.count(slot); + if (cnt < min_count) continue; + const uint32_t p1 = pairs_.left(slot); + if (count1_[p1] == 0) continue; + const Sym& s1 = syms_[p1]; + if (s1.len >= cap_) continue; // cannot be extended + const Sym s3 = Concat(s1, syms_[pairs_.right(slot)], cap_); + cands_.AddOrInc(s3, static_cast(cnt) * s3.len); + } + } + + std::vector& cs = cands_.cands(); + // Highest score first; V5 for the tie-break. + std::sort(cs.begin(), cs.end(), [](const Cand& a, const Cand& b) { + if (a.score != b.score) return a.score > b.score; + if (a.sym.len != b.sym.len) return a.sym.len < b.sym.len; + return std::memcmp(a.sym.b, b.sym.b, a.sym.len) < 0; + }); + + ResetTable(); + for (const Cand& c : cs) { + if (syms_.size() >= cfg_.max_symbols) break; + AddSym(c.sym); + } + } + + Tokens Emit() const { + Tokens t; + t.bytes.reserve(kCodeBase + (syms_.size() - kCodeBase) * cap_); + t.offsets.reserve(syms_.size() + 1); + t.offsets.push_back(0); + for (const Sym& s : syms_) { + t.bytes.insert(t.bytes.end(), s.b, s.b + s.len); + t.offsets.push_back(static_cast(t.bytes.size())); + } + return t; + } + + const Config& cfg_; + const size_t cap_; // effective max symbol length + + std::vector buf_; // sample backing store + std::vector> lines_; // sample rows + + std::vector syms_; // 0..255 literals, then learned symbols + SymbolIndex index_; + std::vector count1_; + PairCounts pairs_; + CandSet cands_; +}; + +} // namespace + +Tokens Train(const uint8_t* bytes, const uint32_t* offsets, size_t num_rows, + const Config& cfg) { + Trainer t(cfg); + return t.Run(bytes, offsets, num_rows); +} + +} // namespace parquet::fsst16 diff --git a/cpp/src/parquet/onpair/fsst16.h b/cpp/src/parquet/onpair/fsst16.h new file mode 100644 index 000000000000..4c7358bbd252 --- /dev/null +++ b/cpp/src/parquet/onpair/fsst16.h @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// FSST's symbol-table training algorithm, lifted to a 16-bit code space. +// +// Reference: P. Boncz, T. Neumann, V. Leis, "FSST: Fast Random Access String +// Compression", VLDB 2020. The vendored reference implementation trains a +// 255-symbol table addressed by one output byte, with one code reserved to +// escape a literal byte. This trains the same way but over a table addressed by +// two output bytes, which is what a like-for-like comparison against a +// 16-bit-code dictionary codec needs. +// +// WHY THIS IS A SEPARATE IMPLEMENTATION, NOT A PARAMETER +// +// The reference cannot be widened in place. Two of its structures are tied to +// the narrow code space: +// +// * The pair-frequency counter is a dense code-by-code matrix. It is a few +// hundred KiB at a 9-bit code space and tens of GiB at a 16-bit one, so a +// 16-bit table needs a sparse counter and a candidate-generation loop that +// walks occupied entries instead of the full square. +// * A symbol is stored in a single 64-bit word, which caps it at 8 bytes. A +// 16-byte symbol needs a wider representation, and the byte-at-a-time +// longest-match index built around that word has to be replaced. +// +// The vendored reference is therefore left untouched, so the 8-bit baseline it +// produces stays exactly what it was. +// +// CODE SPACE +// +// Codes 0..255 are the literal bytes; codes 256 and up are learned symbols. +// This is the natural reading of the reference's own escape mechanism at two +// bytes per code: there, escaping a literal costs a code plus the byte, so the +// trainer works hard to keep literals rare. Here a literal costs one code, the +// same as any symbol, so escapes disappear and the 256 single bytes are simply +// always resident. Every emitted code is the same fixed width, so the output +// size is exactly two bytes times the number of codes, and the trainer's +// objective reduces to emitting as few codes as possible. +// +// TRAINING SHAPE PRESERVED FROM THE REFERENCE +// +// Progressive sampling over five rounds at increasing sample fractions; each +// round compresses the sample with the current table while counting single- +// symbol and adjacent-pair frequencies; candidate symbols are the counted +// symbols plus every counted pair concatenated; a candidate's score is its +// count times its length; single-byte candidates are scored eight times up; +// candidates below a round-scaled minimum count are discarded; the table is +// cleared and refilled from the highest-scoring candidates each round; the +// round with the best measured gain is kept and rebuilt from its own counts at +// the end. +// +// DEVIATIONS, and why each is forced or harmless +// +// V1. Sparse pair counter. Open-addressed rather than a dense matrix, for the +// reason above. Counts are 32-bit and do not saturate, where the +// reference's pair counts saturate at twelve bits. A pair frequent enough +// to saturate is selected either way, so this can only change the +// relative order of two already-selected candidates. +// V2. Candidate generation walks the occupied pair entries rather than nesting +// a right-code loop inside a left-code loop. The set of candidates is the +// same; the order in which they are first seen is not, which matters only +// for ties. +// V3. No terminator byte. The reference picks the least frequent byte as a +// terminator, forces it into every table, and refuses to build a +// multi-byte symbol containing it, so that its match loop can read past +// the end of a string. This bounds-checks its match loop instead, which +// removes the special case entirely. +// V4. Single-byte candidates are scored and ranked but never admitted, since +// the 256 literals are already resident. The eight-times promotion is kept +// so the pair-generation path sees the same scores, but it cannot change +// the table. +// V5. Ties in candidate score are broken by shorter-first then lexicographic +// byte order, rather than by the reference's numeric ordering of the +// symbol's packed word. Both are arbitrary; a total order is all that is +// needed for a deterministic table. +// V6. No code renumbering at the end. The reference renumbers so that its +// most frequent symbols land in the range addressable by a single byte; +// with a fixed two-byte code, code order cannot affect the output size. +// +// The trained table is emitted as a token list, so the tokenizer and decoder of +// the 16-bit dictionary codec it is being compared against can consume it +// directly. That is deliberate: the parsing pass and the decode pass are then +// literally the same code for both, and every difference that remains is a +// difference in how the table was chosen. +// +// NOT a production encoder - this is a benchmark artifact. +// +// Little-endian hosts only. + +#pragma once + +#include +#include +#include + +namespace parquet::fsst16 { + +/// Longest symbol the reference can represent, and the cap this trainer allows +/// as an upper bound on `Config::max_symbol_len`. +constexpr size_t kMaxSymbolLen = 16; + +struct Config { + /// Longest symbol the trainer may build. 8 is the reference's own cap and + /// isolates the effect of the wider code; 16 removes that cap so the only + /// remaining difference from a 16-byte dictionary codec is the training. + int max_symbol_len = 8; + /// Bytes of the column the trainer looks at. The reference's default is 16 + /// KiB regardless of column size. + size_t sample_target = size_t{1} << 14; + /// Table ceiling, counting the 256 resident literals. + size_t max_symbols = size_t{1} << 16; + /// Sample-selection seed. The reference's constant. + uint64_t seed = 4637947; +}; + +/// A trained table as a flat token list: the 256 literals in code order first, +/// then the learned symbols. Token id is the code. +struct Tokens { + std::vector bytes; + std::vector offsets; // length num_tokens + 1 + + size_t num_tokens() const { return offsets.empty() ? 0 : offsets.size() - 1; } +}; + +/// Train a table against (bytes, offsets). `offsets` has length num_rows + 1; +/// row i is bytes[offsets[i]..offsets[i+1]]. +Tokens Train(const uint8_t* bytes, const uint32_t* offsets, size_t num_rows, + const Config& cfg); + +} // namespace parquet::fsst16 diff --git a/cpp/src/parquet/onpair/fsst_onpair_benchmark.cc b/cpp/src/parquet/onpair/fsst_onpair_benchmark.cc index 50c62f5bda9c..70a508de3ad9 100644 --- a/cpp/src/parquet/onpair/fsst_onpair_benchmark.cc +++ b/cpp/src/parquet/onpair/fsst_onpair_benchmark.cc @@ -45,10 +45,12 @@ #include "fsst.h" #include "parquet/onpair/bench_common.h" +#include "parquet/onpair/fsst16.h" #include "parquet/onpair/onpair.h" #include "parquet/onpair/prefix_plus.h" namespace op = parquet::onpair; +namespace f16 = parquet::fsst16; namespace { @@ -254,7 +256,8 @@ Measured RunLz4(const Corpus& c) { // OnPair -Measured RunOnPair(const Corpus& c, uint8_t bits, double threshold) { +Measured RunOnPair(const Corpus& c, uint8_t bits, double threshold, size_t* out_tokens = nullptr, + size_t* out_max_len = nullptr) { op::Config cfg; cfg.max_dict_bits = bits; cfg.threshold_fraction = threshold; @@ -264,6 +267,8 @@ Measured RunOnPair(const Corpus& c, uint8_t bits, double threshold) { op::Column col = op::Compress(c.bytes.data(), c.raw_bytes(), c.offsets.data(), n, cfg); Measured m; m.label = "OnPair" + std::to_string(bits); + if (out_tokens != nullptr) *out_tokens = col.dict.num_tokens(); + if (out_max_len != nullptr) *out_max_len = col.dict.max_token_len; // Realistic bit-packed accounting: codes packed at the true code width for the // trained dictionary (not a fixed u16), dictionary offsets bit-packed, and the // shared per-row length array (in place of the OnPair code-offset array). @@ -378,6 +383,74 @@ Measured RunOnPairAuto(const Corpus& c, double threshold) { return m; } +// FSST's training algorithm at a 16-bit code space +// +// Same page layout, same parsing pass and same decode kernel as OnPair16 - only +// the table is built differently (see fsst16.h). That is the point of the row: +// with the format and both hot loops held identical, the difference between this +// and OnPair16 is attributable to table construction and nothing else, so it is +// charged by OnPairSize exactly as OnPair16 is. +// +// `max_symbol_len` 8 is FSST's own cap, which isolates the effect of the wider +// code; 16 removes the cap so nothing but the training differs from OnPair16. +Measured RunFsst16(const Corpus& c, int max_symbol_len, size_t sample_target = 0, + const char* suffix = "", size_t* out_tokens = nullptr, + size_t* out_max_len = nullptr) { + size_t n = c.n_rows(); + f16::Config cfg; + cfg.max_symbol_len = max_symbol_len; + if (sample_target != 0) cfg.sample_target = sample_target; + + auto build = [&] { + f16::Tokens t = f16::Train(c.bytes.data(), c.offsets.data(), n, cfg); + return op::CompressWithTokens(c.bytes.data(), c.offsets.data(), n, t.bytes, t.offsets); + }; + + op::Column col = build(); + Measured m; + m.label = "FSST16-" + std::to_string(max_symbol_len) + "B" + suffix; + m.compressed_bytes = OnPairSize(col, c); + if (out_tokens != nullptr) *out_tokens = col.dict.num_tokens(); + if (out_max_len != nullptr) *out_max_len = col.dict.max_token_len; + + std::vector enc; + for (int it = 0; it < kEncodeIters; ++it) { + auto t0 = Clock::now(); + op::Column tmp = build(); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(tmp.codes.size()) : "memory"); + enc.push_back(Mib(c.raw_bytes()) / dt); + } + m.encode_mibs = Median(std::move(enc)); + + size_t code_bits = IndexBits(col.dict.num_tokens()); + size_t cap = op::DecodedLen(col) + op::kDecodePadding; + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), code_bits); + { + std::vector out(cap, 0); + size_t w = + op::DecompressPacked(col.dict, packed.data(), col.codes.size(), code_bits, out.data()); + if (w != c.raw_bytes() || std::memcmp(out.data(), c.bytes.data(), c.raw_bytes()) != 0) { + std::fprintf(stderr, "FSST16-%dB packed roundtrip mismatch on %s (w=%zu raw=%zu)\n", + max_symbol_len, c.name.c_str(), w, c.raw_bytes()); + std::abort(); + } + } + std::vector dec_r; + for (int it = 0; it < kDecodeIters; ++it) { + std::vector out(cap, 0); + auto t0 = Clock::now(); + size_t w = + op::DecompressPacked(col.dict, packed.data(), col.codes.size(), code_bits, out.data()); + double dt = std::chrono::duration(Clock::now() - t0).count(); + asm volatile("" ::"r"(w) : "memory"); + dec_r.push_back(Mib(c.raw_bytes()) / dt); + } + m.decode_mibs = Median(std::move(dec_r)); + return m; +} + // dedup-then-OnPair // // The layout a real columnar format uses for repetitive columns: encode the @@ -988,6 +1061,12 @@ int main(int argc, char** argv) { return 1; } + // ONPAIR_BENCH_CORE restricts the run to the FSST-vs-OnPair table set: the two + // FSST16 variants next to the FSST8 baseline and the two OnPair rows, with the + // cascade and dedup codecs skipped so a comparison run does not pay for rows it + // does not report. Same binary and therefore the same accounting either way. + const bool core_only = std::getenv("ONPAIR_BENCH_CORE") != nullptr; + std::printf("%-26s %10s %10s %7s %9s %9s\n", "corpus", "rows", "raw MiB", "ratio", "enc MiB/s", "dec MiB/s"); std::printf("%s\n", std::string(90, '-').c_str()); @@ -997,19 +1076,43 @@ int main(int argc, char** argv) { double threshold = ThresholdFor(c.name); Measured fsst = RunFsst(c); + size_t t8 = 0, l8 = 0, t16 = 0, l16 = 0, tfull = 0, lfull = 0; + Measured f16_8 = RunFsst16(c, 8, 0, "", &t8, &l8); + Measured f16_16 = RunFsst16(c, 16, 0, "", &t16, &l16); + // Same 16-byte cap, but trained on 4 MiB instead of the reference's fixed 16 + // KiB. The rows above look at 16 KiB no matter how large the column is, which + // caps how many distinct symbols the trainer can even see evidence for; this + // row separates that sampling choice from the training algorithm itself. 4 MiB + // is 256 times the reference sample and enough to fill a 16-bit table, while + // staying bounded enough to time. + Measured f16_m = RunFsst16(c, 16, size_t{4} << 20, "-4M", &tfull, &lfull); + size_t op16_tokens = 0, op16_max_len = 0; + Measured op16 = RunOnPair(c, 16, threshold, &op16_tokens, &op16_max_len); + Measured opauto = RunOnPairAuto(c, threshold); + + std::printf("%-26s %10zu %10.2f\n", c.name.c_str(), c.n_rows(), Mib(c.raw_bytes())); + auto emit = [&](const Measured& m) { + double ratio = static_cast(c.raw_bytes()) / static_cast(m.compressed_bytes); + std::printf(" %-24s %10s %10.2f %7.3fx %9.1f %9.1f\n", m.label.c_str(), "", + Mib(m.compressed_bytes), ratio, m.encode_mibs, m.decode_mibs); + }; + + if (core_only) { + for (const Measured* m : {&fsst, &f16_8, &f16_16, &f16_m, &op16, &opauto}) emit(*m); + std::printf(" -> tables: FSST16-8B %zu tokens/max %zu, FSST16-16B %zu/%zu, " + "FSST16-16B-4M %zu/%zu, OnPair16 %zu/%zu\n\n", + t8, l8, t16, l16, tfull, lfull, op16_tokens, op16_max_len); + continue; + } Measured zstd1 = RunZstd(c, 1); Measured lz4 = RunLz4(c); - Measured op16 = RunOnPair(c, 16, threshold); - Measured opauto = RunOnPairAuto(c, threshold); Measured op16d = RunOnPairDedup(c, 16, threshold); Measured fsstp = RunFsstPlus(c); Measured oppl = RunOnPairPlus(c, threshold); - std::printf("%-26s %10zu %10.2f\n", c.name.c_str(), c.n_rows(), Mib(c.raw_bytes())); - for (const Measured* m : {&fsst, &zstd1, &lz4, &op16, &opauto, &op16d, &fsstp, &oppl}) { - double ratio = static_cast(c.raw_bytes()) / static_cast(m->compressed_bytes); - std::printf(" %-24s %10s %10.2f %7.3fx %9.1f %9.1f\n", m->label.c_str(), "", - Mib(m->compressed_bytes), ratio, m->encode_mibs, m->decode_mibs); + for (const Measured* m : + {&fsst, &f16_8, &f16_16, &f16_m, &zstd1, &lz4, &op16, &opauto, &op16d, &fsstp, &oppl}) { + emit(*m); } double r_fsst = static_cast(c.raw_bytes()) / fsst.compressed_bytes; double r_zstd = static_cast(c.raw_bytes()) / zstd1.compressed_bytes; diff --git a/cpp/src/parquet/onpair/onpair.cc b/cpp/src/parquet/onpair/onpair.cc index bdb0c1201ac0..a71356a40035 100644 --- a/cpp/src/parquet/onpair/onpair.cc +++ b/cpp/src/parquet/onpair/onpair.cc @@ -737,6 +737,26 @@ Column Compress(const uint8_t* bytes, size_t /*bytes_len*/, const uint32_t* offs return col; } +Column CompressWithTokens(const uint8_t* bytes, const uint32_t* offsets, size_t num_rows, + const std::vector& token_bytes, + const std::vector& token_offsets) { + std::vector sorted_bytes; + std::vector sorted_offsets; + SortTokens(token_bytes, token_offsets, &sorted_bytes, &sorted_offsets); + PadRaw(&sorted_bytes, sorted_offsets); + + Column col; + col.dict.bytes = std::move(sorted_bytes); + col.dict.offsets = std::move(sorted_offsets); + col.dict.RecomputeMaxTokenLen(); + LongestPrefixMatcher lpm = LongestPrefixMatcher::FromDictionary(col.dict); + + col.codes.reserve(num_rows == 0 ? 0 : offsets[num_rows]); + col.row_offsets.reserve(num_rows + 1); + EncodeStrings(bytes, offsets, num_rows, lpm, &col.codes, &col.row_offsets); + return col; +} + size_t DecodedLen(const Column& col) { size_t sum = 0; for (uint16_t c : col.codes) sum += col.dict.token_len(c); diff --git a/cpp/src/parquet/onpair/onpair.h b/cpp/src/parquet/onpair/onpair.h index 741dadde9fb0..1e3af8e6f1d7 100644 --- a/cpp/src/parquet/onpair/onpair.h +++ b/cpp/src/parquet/onpair/onpair.h @@ -185,6 +185,20 @@ struct EncodeProfile { Column Compress(const uint8_t* bytes, size_t bytes_len, const uint32_t* offsets, size_t num_rows, const Config& cfg, EncodeProfile* profile = nullptr); +/// Tokenize every row against a token set trained elsewhere, skipping OnPair's +/// own training. `token_bytes`/`token_offsets` are a raw token list in the same +/// layout as CompactDictionary but without the read padding; the returned +/// column's dictionary is its canonical (sorted, padded) form, so token ids are +/// reassigned and the caller's numbering is not preserved. +/// +/// This exists so an alternative dictionary trainer can be measured against +/// OnPair's with the parsing pass and the decode pass held literally identical. +/// The token set must contain all 256 single bytes, which is what lets both +/// tokenize without an escape mechanism. +Column CompressWithTokens(const uint8_t* bytes, const uint32_t* offsets, size_t num_rows, + const std::vector& token_bytes, + const std::vector& token_offsets); + /// Exact decoded byte length of the whole column (sum of token lengths). size_t DecodedLen(const Column& col); diff --git a/cpp/src/parquet/onpair/verify_roundtrip.cc b/cpp/src/parquet/onpair/verify_roundtrip.cc index b9bcfe7efa9c..363eb2cbd4f9 100644 --- a/cpp/src/parquet/onpair/verify_roundtrip.cc +++ b/cpp/src/parquet/onpair/verify_roundtrip.cc @@ -13,21 +13,28 @@ // License for the specific language governing permissions and limitations // under the License. -// Visible round-trip proof for the OnPair port: decode both plain OnPair16 and -// OnPair16-dedup and check EVERY row equals the original bytes, then print a few -// concrete original -> decoded samples so a human can eyeball the recovery. +// Visible round-trip proof for the OnPair port and for the FSST16 trainer it is +// compared against: decode every configuration and check EVERY row equals the +// original bytes, then print a few concrete original -> decoded samples so a +// human can eyeball the recovery. // -// Every dictionary budget in the valid 9..16 range is checked, not just 16: the -// packed decode loop is templated on the code width, and OnPair-auto picks a -// width per column, so verifying only 16 would leave the width the benchmarks -// actually report unverified. The merge threshold comes from bench_common.h so -// the dictionary trained here is the one the benchmarks measure. +// The FSST16 rows are decoded twice, once through the whole-column path and once +// through the bit-packed path at the code width the trained table actually needs, +// because the packed loop is templated on that width and a table of a few hundred +// tokens exercises a far narrower one than OnPair ever produces. // -// Exits non-zero if any row of any corpus at any width fails to round-trip, so -// this can gate a run. +// For OnPair, every dictionary budget in the valid 9..16 range is checked, not +// just 16: OnPair-auto picks a width per column, so verifying only 16 would leave +// the width the benchmarks actually report unverified. The merge threshold comes +// from bench_common.h so the dictionary trained here is the one the benchmarks +// measure. +// +// Exits non-zero if any row of any corpus in any configuration or at any width +// fails to round-trip, so this can gate a benchmark run. // // Build (one line): g++ -std=c++17 -O2 -Icpp/src -// cpp/src/parquet/onpair/onpair.cc cpp/src/parquet/onpair/verify_roundtrip.cc -o /tmp/verify +// cpp/src/parquet/onpair/onpair.cc cpp/src/parquet/onpair/fsst16.cc +// cpp/src/parquet/onpair/verify_roundtrip.cc -o /tmp/verify // Run: /tmp/verify bench-fsst-onpair/corpora/tpch_l_shipmode.txt [more files...] #include @@ -40,9 +47,11 @@ #include #include "parquet/onpair/bench_common.h" +#include "parquet/onpair/fsst16.h" #include "parquet/onpair/onpair.h" namespace op = parquet::onpair; +namespace f16 = parquet::fsst16; namespace { @@ -214,6 +223,50 @@ bool VerifyOnPair(const Corpus& c, uint8_t bits, double threshold, bool show_sam return paths && bad == 0; } +// FSST's training algorithm at a 16-bit code space, decoded through OnPair's own +// path. The trained token list is handed to the encoder's train-free entry point, +// so the parsing pass and the decode kernel are the same code OnPair16 uses and +// only the table differs. +bool VerifyFsst16(const Corpus& c, int max_symbol_len, size_t sample_target, const char* tag, + bool show_samples) { + f16::Config cfg; + cfg.max_symbol_len = max_symbol_len; + if (sample_target != 0) cfg.sample_target = sample_target; + f16::Tokens t = f16::Train(c.bytes.data(), c.offsets.data(), c.rows(), cfg); + op::Column col = + op::CompressWithTokens(c.bytes.data(), c.offsets.data(), c.rows(), t.bytes, t.offsets); + + std::vector out(op::DecodedLen(col) + op::kDecodePadding, 0); + size_t dn = op::DecompressInto(col, out.data()); + long bad_at; + size_t bad = CheckPerRow(c, out.data(), dn, &bad_at); + + // The width a stored format would pack these codes at, which is what the + // benchmark charges for and therefore what has to decode correctly. + size_t nt = col.dict.num_tokens(); + size_t code_bits = 1; + while ((size_t{1} << code_bits) < nt) ++code_bits; + std::vector cw(col.codes.begin(), col.codes.end()); + std::vector packed = op::PackValues(cw.data(), cw.size(), code_bits); + std::vector pout(op::DecodedLen(col) + op::kDecodePadding, 0); + size_t pn = + op::DecompressPacked(col.dict, packed.data(), col.codes.size(), code_bits, pout.data()); + long pbad_at; + size_t pbad = CheckPerRow(c, pout.data(), pn, &pbad_at); + + std::printf(" %-15s: %zu/%zu rows exact, packed %zu/%zu (%zu tokens, %zub codes, " + "max token %zu) %s\n", + tag, c.rows() - bad, c.rows(), c.rows() - pbad, c.rows(), nt, code_bits, + col.dict.max_token_len, (bad == 0 && pbad == 0) ? "[OK]" : "[FAIL]"); + if (bad != 0) std::printf(" first mismatching row: %ld\n", bad_at); + if (pbad != 0) std::printf(" first mismatching packed row: %ld\n", pbad_at); + if (show_samples) Samples(c, pout.data()); + // An FSST-trained table at an 8-byte cap stays far smaller than OnPair's, so this + // is where the narrow end of the width dispatch gets exercised. + bool fsst_paths = VerifyDecodePaths(col, " ↳ paths"); + return bad == 0 && pbad == 0 && fsst_paths; +} + // The distinct-value set a dedup cascade OnPairs, plus the per-row ids into it. // Built once per corpus and reused across widths -- deduplicating 500k rows is // far more expensive than the training pass being verified. @@ -295,6 +348,14 @@ int main(int argc, char** argv) { double threshold = bench::ThresholdFor(std::filesystem::path(argv[a]).stem().string()); std::printf("\n%s (%zu rows, %.2f MiB, threshold %.2f)\n", argv[a], c.rows(), c.bytes.size() / (1024.0 * 1024.0), threshold); + // FSST16 at its native symbol cap, at OnPair's cap, and at a sample large + // enough to actually fill a 16-bit table. The last one is the config whose + // dictionary is big enough to reach the wide packed-decode loops. Its code + // width follows from the trained table, so these run once, not per budget. + if (!VerifyFsst16(c, 8, 0, "FSST16-8B", false)) ++failures; + if (!VerifyFsst16(c, 16, 0, "FSST16-16B", false)) ++failures; + if (!VerifyFsst16(c, 16, size_t{4} << 20, "FSST16-16B-4M", true)) ++failures; + rows_checked += 6 * c.rows(); // 3 configs, each checked unpacked and packed Distinct d = BuildDistinct(c); for (uint8_t bits = 9; bits <= 16; ++bits) { // Samples are the human-readable proof; print them once, at the width the