Skip to content

Commit d512dd2

Browse files
authored
GH-34078: [C++][Parquet] Minor API improvements for BloomFilter (#33995)
### Rationale for this change * adding a interface `OptimalNumOfBytes`, because `OptimalNumOfBits` is confusing... * BloomFilter accept a `MemoryPool` as input argument ### What changes are included in this PR? ### Are these changes tested? They're already tested... ### Are there any user-facing changes? No. (But user may misuse `BloomFilter::Init` previously) * Closes: #34078 * Closes: #34078 Authored-by: mwish <maplewish117@gmail.com> Signed-off-by: Will Jones <willjones127@gmail.com>
1 parent 9cb6fd6 commit d512dd2

3 files changed

Lines changed: 51 additions & 32 deletions

File tree

cpp/src/parquet/bloom_filter.cc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@
3030
namespace parquet {
3131
constexpr uint32_t BlockSplitBloomFilter::SALT[kBitsSetPerBlock];
3232

33-
BlockSplitBloomFilter::BlockSplitBloomFilter()
34-
: pool_(::arrow::default_memory_pool()),
33+
BlockSplitBloomFilter::BlockSplitBloomFilter(::arrow::MemoryPool* pool)
34+
: pool_(pool),
3535
hash_strategy_(HashStrategy::XXHASH),
3636
algorithm_(Algorithm::BLOCK),
3737
compression_strategy_(CompressionStrategy::UNCOMPRESSED) {}

cpp/src/parquet/bloom_filter.h

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,10 @@ class PARQUET_EXPORT BloomFilter {
122122
class PARQUET_EXPORT BlockSplitBloomFilter : public BloomFilter {
123123
public:
124124
/// The constructor of BlockSplitBloomFilter. It uses XXH64 as hash function.
125-
BlockSplitBloomFilter();
125+
///
126+
/// \param pool memory pool to use.
127+
explicit BlockSplitBloomFilter(
128+
::arrow::MemoryPool* pool = ::arrow::default_memory_pool());
126129

127130
/// Initialize the BlockSplitBloomFilter. The range of num_bytes should be within
128131
/// [kMinimumBloomFilterBytes, kMaximumBloomFilterBytes], it will be
@@ -152,6 +155,19 @@ class PARQUET_EXPORT BlockSplitBloomFilter : public BloomFilter {
152155
/// @param fpp The false positive probability.
153156
/// @return it always return a value between kMinimumBloomFilterBytes and
154157
/// kMaximumBloomFilterBytes, and the return value is always a power of 2
158+
static uint32_t OptimalNumOfBytes(uint32_t ndv, double fpp) {
159+
uint32_t optimal_num_of_bits = OptimalNumOfBits(ndv, fpp);
160+
DCHECK(::arrow::bit_util::IsMultipleOf8(optimal_num_of_bits));
161+
return optimal_num_of_bits >> 3;
162+
}
163+
164+
/// Calculate optimal size according to the number of distinct values and false
165+
/// positive probability.
166+
///
167+
/// @param ndv The number of distinct values.
168+
/// @param fpp The false positive probability.
169+
/// @return it always return a value between kMinimumBloomFilterBytes * 8 and
170+
/// kMaximumBloomFilterBytes * 8, and the return value is always a power of 16
155171
static uint32_t OptimalNumOfBits(uint32_t ndv, double fpp) {
156172
DCHECK(fpp > 0.0 && fpp < 1.0);
157173
const double m = -8.0 * ndv / log(1 - pow(fpp, 1.0 / 8));

cpp/src/parquet/bloom_filter_test.cc

Lines changed: 32 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ TEST(FPPTest, TestBloomFilter) {
159159

160160
std::vector<std::string> members;
161161
BlockSplitBloomFilter bloom_filter;
162-
bloom_filter.Init(BlockSplitBloomFilter::OptimalNumOfBits(total_count, fpp));
162+
bloom_filter.Init(BlockSplitBloomFilter::OptimalNumOfBytes(total_count, fpp));
163163

164164
// Insert elements into the Bloom filter
165165
for (int i = 0; i < total_count; i++) {
@@ -249,36 +249,39 @@ TEST(CompatibilityTest, TestBloomFilter) {
249249
// Also it is used to test whether OptimalNumOfBits returns value between
250250
// [MINIMUM_BLOOM_FILTER_SIZE, MAXIMUM_BLOOM_FILTER_SIZE].
251251
TEST(OptimalValueTest, TestBloomFilter) {
252-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(256, 0.01), UINT32_C(4096));
253-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(512, 0.01), UINT32_C(8192));
254-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(1024, 0.01), UINT32_C(16384));
255-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(2048, 0.01), UINT32_C(32768));
256-
257-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(200, 0.01), UINT32_C(2048));
258-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(300, 0.01), UINT32_C(4096));
259-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(700, 0.01), UINT32_C(8192));
260-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(1500, 0.01), UINT32_C(16384));
261-
262-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(200, 0.025), UINT32_C(2048));
263-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(300, 0.025), UINT32_C(4096));
264-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(700, 0.025), UINT32_C(8192));
265-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(1500, 0.025), UINT32_C(16384));
266-
267-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(200, 0.05), UINT32_C(2048));
268-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(300, 0.05), UINT32_C(4096));
269-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(700, 0.05), UINT32_C(8192));
270-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(1500, 0.05), UINT32_C(16384));
252+
auto testOptimalNumEstimation = [](uint32_t ndv, double fpp, uint32_t num_bits) {
253+
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(ndv, fpp), num_bits);
254+
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBytes(ndv, fpp), num_bits / 8);
255+
};
256+
257+
testOptimalNumEstimation(256, 0.01, UINT32_C(4096));
258+
testOptimalNumEstimation(512, 0.01, UINT32_C(8192));
259+
testOptimalNumEstimation(1024, 0.01, UINT32_C(16384));
260+
testOptimalNumEstimation(2048, 0.01, UINT32_C(32768));
261+
262+
testOptimalNumEstimation(200, 0.01, UINT32_C(2048));
263+
testOptimalNumEstimation(300, 0.01, UINT32_C(4096));
264+
testOptimalNumEstimation(700, 0.01, UINT32_C(8192));
265+
testOptimalNumEstimation(1500, 0.01, UINT32_C(16384));
266+
267+
testOptimalNumEstimation(200, 0.025, UINT32_C(2048));
268+
testOptimalNumEstimation(300, 0.025, UINT32_C(4096));
269+
testOptimalNumEstimation(700, 0.025, UINT32_C(8192));
270+
testOptimalNumEstimation(1500, 0.025, UINT32_C(16384));
271+
272+
testOptimalNumEstimation(200, 0.05, UINT32_C(2048));
273+
testOptimalNumEstimation(300, 0.05, UINT32_C(4096));
274+
testOptimalNumEstimation(700, 0.05, UINT32_C(8192));
275+
testOptimalNumEstimation(1500, 0.05, UINT32_C(16384));
271276

272277
// Boundary check
273-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(4, 0.01), UINT32_C(256));
274-
EXPECT_EQ(BlockSplitBloomFilter::OptimalNumOfBits(4, 0.25), UINT32_C(256));
275-
276-
EXPECT_EQ(
277-
BlockSplitBloomFilter::OptimalNumOfBits(std::numeric_limits<uint32_t>::max(), 0.01),
278-
UINT32_C(1073741824));
279-
EXPECT_EQ(
280-
BlockSplitBloomFilter::OptimalNumOfBits(std::numeric_limits<uint32_t>::max(), 0.25),
281-
UINT32_C(1073741824));
278+
testOptimalNumEstimation(4, 0.01, BlockSplitBloomFilter::kMinimumBloomFilterBytes * 8);
279+
testOptimalNumEstimation(4, 0.25, BlockSplitBloomFilter::kMinimumBloomFilterBytes * 8);
280+
281+
testOptimalNumEstimation(std::numeric_limits<uint32_t>::max(), 0.01,
282+
BlockSplitBloomFilter::kMaximumBloomFilterBytes * 8);
283+
testOptimalNumEstimation(std::numeric_limits<uint32_t>::max(), 0.25,
284+
BlockSplitBloomFilter::kMaximumBloomFilterBytes * 8);
282285
}
283286

284287
// The test below is plainly copied from parquet-mr and serves as a basic sanity

0 commit comments

Comments
 (0)