Skip to content

Commit 400a6d9

Browse files
haziqishereraulcd
andauthored
GH-49927: [Python][Parquet] Expose bloom_filter_offset and bloom_filter_length to Python in column chunk metadata (#49926)
### Rationale for this change `ColumnChunkMetaData.to_dict()` method omits `bloom_filter_offset` and `bloom_filter_length` even when a bloom filter is written to the Parquet file. Users cannot programmatically verify bloom filter presence via the Python metadata API without resorting to file size comparison. ### What changes are included in this PR? 1. `python/pyarrow/includes/libparquet.pxd`: Declare `bloom_filter_offset()` and `bloom_filter_length()` (both optional[int64_t]) on `CColumnChunkMetaData`. This is to expose the existing C++ methods to Cython. 2. `python/pyarrow/_parquet.pyx`: Add `bloom_filter_offset` and `bloom_filter_length` properties to `ColumnChunkMetaData` (returns int when set, None otherwise). Add both fields to `to_dict()` and `__repr__`. 3. `python/pyarrow/tests/parquet/test_metadata.py`: Add `test_bloom_filter_offset_in_metadata` verifying that columns with a bloom filter expose non-None integer values and that `to_dict()` contains the keys, while columns without a bloom filter return None. ### Are these changes tested? Yes. `test_bloom_filter_offset_in_metadata` in test_metadata.py covers: - Column with bloom filter: bloom_filter_offset and bloom_filter_length are non-None integers - Column without bloom filter: both return None - Both keys present in to_dict() output <img width="863" height="215" alt="image" src="https://github.com/user-attachments/assets/d465d6bd-55d1-4c5f-9f11-6a60b3bf1cbe" /> Here is closer look on the logic output: <img width="464" height="424" alt="image" src="https://github.com/user-attachments/assets/6e4810f2-c1c0-41ea-b559-00f99d42e2c4" /> output: ```python col_a bloom_filter_offset: 10699 col_a bloom_filter_length: 1040 col_b bloom_filter_offset: None col_b bloom_filter_length: None col_a to_dict(): {'file_offset': 0, 'file_path': '', 'physical_type': 'BYTE_ARRAY', 'num_values': 1000, 'path_in_schema': 'a', 'is_stats_set': True, 'statistics': {'has_min_max': True, 'min': 'id_0', 'max': 'id_999', 'null_count': 0, 'distinct_count': None, 'num_values': 1000, 'physical_type': 'BYTE_ARRAY'}, 'geo_statistics': None, 'compression': 'SNAPPY', 'encodings': ('PLAIN', 'RLE', 'RLE_DICTIONARY'), 'has_dictionary_page': True, 'dictionary_page_offset': 4, 'data_page_offset': 4035, 'total_compressed_size': 5336, 'total_uncompressed_size': 11208, 'bloom_filter_offset': 10699, 'bloom_filter_length': 1040} ``` * GitHub Issue: #49927 Lead-authored-by: Haziq Hakimi Mazlisham <95128744+haziqishere@users.noreply.github.com> Co-authored-by: Raúl Cumplido <raulcumplido@gmail.com> Signed-off-by: Raúl Cumplido <raulcumplido@gmail.com>
1 parent a0d2885 commit 400a6d9

4 files changed

Lines changed: 61 additions & 2 deletions

File tree

docs/source/python/parquet.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,8 @@ such as the row groups and column chunk metadata and statistics:
332332
data_page_offset: 36
333333
total_compressed_size: 106
334334
total_uncompressed_size: 102
335+
bloom_filter_offset: None
336+
bloom_filter_length: None
335337
336338
Data Type Handling
337339
------------------

python/pyarrow/_parquet.pyx

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -475,7 +475,9 @@ cdef class ColumnChunkMetaData(_Weakrefable):
475475
dictionary_page_offset: {self.dictionary_page_offset}
476476
data_page_offset: {self.data_page_offset}
477477
total_compressed_size: {self.total_compressed_size}
478-
total_uncompressed_size: {self.total_uncompressed_size}"""
478+
total_uncompressed_size: {self.total_uncompressed_size}
479+
bloom_filter_offset: {self.bloom_filter_offset}
480+
bloom_filter_length: {self.bloom_filter_length}"""
479481

480482
def to_dict(self):
481483
"""
@@ -507,7 +509,9 @@ cdef class ColumnChunkMetaData(_Weakrefable):
507509
dictionary_page_offset=self.dictionary_page_offset,
508510
data_page_offset=self.data_page_offset,
509511
total_compressed_size=self.total_compressed_size,
510-
total_uncompressed_size=self.total_uncompressed_size
512+
total_uncompressed_size=self.total_uncompressed_size,
513+
bloom_filter_offset=self.bloom_filter_offset,
514+
bloom_filter_length=self.bloom_filter_length,
511515
)
512516
return d
513517

@@ -645,6 +649,22 @@ cdef class ColumnChunkMetaData(_Weakrefable):
645649
"""Uncompressed size in bytes (int)."""
646650
return self.metadata.total_uncompressed_size()
647651

652+
@property
653+
def bloom_filter_offset(self):
654+
"""Offset of bloom filter relative to beginning of the file (int or None)."""
655+
cdef optional[int64_t] offset = self.metadata.bloom_filter_offset()
656+
if offset.has_value():
657+
return offset.value()
658+
return None
659+
660+
@property
661+
def bloom_filter_length(self):
662+
"""Length of bloom filter in bytes (int or None)."""
663+
cdef optional[int64_t] length = self.metadata.bloom_filter_length()
664+
if length.has_value():
665+
return length.value()
666+
return None
667+
648668
@property
649669
def has_offset_index(self):
650670
"""Whether the column chunk has an offset index"""

python/pyarrow/includes/libparquet.pxd

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,8 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
370370
int64_t index_page_offset() const
371371
int64_t total_compressed_size() const
372372
int64_t total_uncompressed_size() const
373+
optional[int64_t] bloom_filter_offset() const
374+
optional[int64_t] bloom_filter_length() const
373375
unique_ptr[CColumnCryptoMetaData] crypto_metadata() const
374376
optional[ParquetIndexLocation] GetColumnIndexLocation() const
375377
optional[ParquetIndexLocation] GetOffsetIndexLocation() const

python/pyarrow/tests/parquet/test_metadata.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -796,6 +796,41 @@ def test_column_chunk_key_value_metadata(parquet_test_datadir):
796796
assert key_value_metadata2 is None
797797

798798

799+
def test_bloom_filter_offset_in_metadata():
800+
# ColumnChunkMetaData.to_dict() when a bloom filter is written.
801+
table = pa.table({"a": [f"id_{i}" for i in range(1000)],
802+
"b": list(range(1000))})
803+
804+
buf = pa.BufferOutputStream()
805+
pq.write_table(
806+
table,
807+
buf,
808+
bloom_filter_options={"a": {"ndv": 1000}} # apply bloom filter on col a
809+
)
810+
metadata = pq.read_metadata(pa.BufferReader(buf.getvalue()))
811+
812+
col_a = metadata.row_group(0).column(0) # bloom filter written
813+
col_b = metadata.row_group(0).column(1) # no bloom filter
814+
815+
assert col_a.bloom_filter_offset is not None
816+
assert isinstance(col_a.bloom_filter_offset, int)
817+
assert col_a.bloom_filter_length is not None
818+
assert isinstance(col_a.bloom_filter_length, int)
819+
820+
assert col_b.bloom_filter_offset is None
821+
assert col_b.bloom_filter_length is None
822+
823+
d = col_a.to_dict()
824+
assert "bloom_filter_offset" in d
825+
assert "bloom_filter_length" in d
826+
assert d["bloom_filter_offset"] == col_a.bloom_filter_offset
827+
assert d["bloom_filter_length"] == col_a.bloom_filter_length
828+
829+
d_no_bloom = col_b.to_dict()
830+
assert d_no_bloom["bloom_filter_offset"] is None
831+
assert d_no_bloom["bloom_filter_length"] is None
832+
833+
799834
def test_internal_class_instantiation():
800835
def msg(c):
801836
return f"Do not call {c}'s constructor directly"

0 commit comments

Comments
 (0)