Skip to content

Commit a81e6c6

Browse files
authored
GH-46179: [Python] Bump index level once if pandas df already contains __index_level_i__ column (#46884)
### Rationale for this change PyArrow adds a `__inex_level_i__` column to the PyArrow table if the pandas dataframe has an unnamed index it wants to preserve. Currently that creates a duplicate, if such a column already exists. ### What changes are included in this PR? Bumping the integer number in the generated column in order to not get any duplicates. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #46179 Authored-by: AlenkaF <frim.alenka@gmail.com> Signed-off-by: Raúl Cumplido <raulcumplido@gmail.com>
1 parent 4fce185 commit a81e6c6

2 files changed

Lines changed: 44 additions & 3 deletions

File tree

python/pyarrow/pandas_compat.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,10 @@ def _index_level_name(index, i, column_names):
378378
if index.name is not None and index.name not in column_names:
379379
return _column_name_to_strings(index.name)
380380
else:
381-
return f'__index_level_{i:d}__'
381+
j = i
382+
while f'__index_level_{j:d}__' in column_names:
383+
j += 1
384+
return f'__index_level_{j:d}__'
382385

383386

384387
def _get_columns_to_convert(df, schema, preserve_index, columns):
@@ -419,7 +422,9 @@ def _get_columns_to_convert(df, schema, preserve_index, columns):
419422
index_descriptors = []
420423
index_column_names = []
421424
for i, index_level in enumerate(index_levels):
422-
name = _index_level_name(index_level, i, column_names)
425+
name = _index_level_name(
426+
index_level, i, column_names + index_column_names
427+
)
423428
if (isinstance(index_level, _pandas_api.pd.RangeIndex) and
424429
preserve_index is None):
425430
descr = _get_range_index_descriptor(index_level)

python/pyarrow/tests/test_pandas.py

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -404,14 +404,50 @@ def test_index_metadata_field_name(self):
404404
assert col3['name'] == col3['field_name']
405405

406406
idx0_descr, foo_descr = js['index_columns']
407-
assert idx0_descr == '__index_level_0__'
407+
# __index_level_0__ exists, unnamed bumped to __index_level_1__
408+
assert idx0_descr == '__index_level_1__'
408409
assert idx0['field_name'] == idx0_descr
409410
assert idx0['name'] is None
410411

411412
assert foo_descr == 'foo'
412413
assert foo['field_name'] == foo_descr
413414
assert foo['name'] == foo_descr
414415

416+
def test_index_level_name_bump(self):
417+
# GH-46179
418+
df = pd.DataFrame(
419+
{"col": [1, 2, 3], "__index_level_0__": [4, 5, 6]},
420+
index=[10, 20, 30],
421+
)
422+
_check_pandas_roundtrip(df, preserve_index=True)
423+
424+
# Explicit test
425+
t = pa.table(df)
426+
expected_schema = pa.schema([
427+
("col", pa.int64()),
428+
("__index_level_0__", pa.int64()),
429+
("__index_level_1__", pa.int64())
430+
])
431+
assert t.schema.equals(expected_schema)
432+
433+
df2 = t.to_pandas()
434+
assert df2.index.equals(pd.Index([10, 20, 30]))
435+
assert df2.ndim == df.ndim == 2
436+
437+
def test_index_level_name_bump_multiindex(self):
438+
# GH-46179
439+
df = pd.DataFrame(
440+
{"col": [1, 2], "__index_level_0__": [3, 4]},
441+
index=pd.MultiIndex.from_arrays(
442+
[[10, 20], [100, 200]], names=[None, None]
443+
),
444+
)
445+
_check_pandas_roundtrip(df, preserve_index=True)
446+
447+
t = pa.Table.from_pandas(df, preserve_index=True)
448+
assert t.schema.names == ['col', '__index_level_0__',
449+
'__index_level_1__', '__index_level_2__']
450+
415451
def test_categorical_column_index(self):
416452
df = pd.DataFrame(
417453
[(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)],

0 commit comments

Comments
 (0)