Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BUG: Fix feature transformation #79

Merged
merged 7 commits into from
Nov 1, 2024
Merged

BUG: Fix feature transformation #79

Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6595566
BUG: Fix feature transform failure when test_id_column=None
AnirudhDagar Oct 31, 2024
c2ed395
assign test_x for else condition
AnirudhDagar Oct 31, 2024
6ecef8c
Handle transformed_test_data similarly
AnirudhDagar Oct 31, 2024
0f82136
BUG: Drop train_id_column for train_x
AnirudhDagar Oct 31, 2024
548816a
BUG: Handle datasets with test set containing label columns
AnirudhDagar Oct 31, 2024
1bd7800
assign test_y if label column available
AnirudhDagar Oct 31, 2024
e2aa9e8
log error in except block; fix typo
AnirudhDagar Nov 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 33 additions & 7 deletions src/autogluon_assistant/transformer/feature_transformers/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def _fit_dataframes(self, train_X: pd.DataFrame, train_y: pd.Series, **kwargs) -
def fit(self, task: TabularPredictionTask) -> "BaseFeatureTransformer":
try:
train_x = task.train_data.drop(
columns=task.columns_in_train_but_not_test + [task.test_id_column],
columns=task.columns_in_train_but_not_test + [task.train_id_column],
errors="ignore",
)
train_y = task.train_data[task.label_column]
Expand All @@ -47,21 +47,47 @@ def _transform_dataframes(self, train_X: pd.DataFrame, test_X: pd.DataFrame) ->
def transform(self, task: TabularPredictionTask) -> TabularPredictionTask:
try:
train_x = task.train_data.drop(
columns=task.columns_in_train_but_not_test + [task.test_id_column],
columns=task.columns_in_train_but_not_test + [task.train_id_column],
errors="ignore",
)
label_column_avaialable_in_test_set = False
if task.label_column in train_x.columns:
# Label Column also present in test data
# requires explicit dropping of the column
# from train set before feature transformation
train_x = train_x.drop(columns=[task.label_column])
train_y = task.train_data[task.label_column]
test_x = task.test_data.drop(columns=[task.test_id_column])

if task.test_id_column in task.test_data.columns:
# Skip if test_id_column is not found
test_x = task.test_data.drop(columns=[task.test_id_column])
else:
test_x = task.test_data
if task.label_column in test_x.columns:
# Label Column also present in test data
# requires explicit dropping of the column
# from test set before feature transformation
label_column_avaialable_in_test_set = True
test_x = test_x.drop(columns=[task.label_column])
test_y = task.test_data[task.label_column]

train_x, test_x = self._transform_dataframes(train_X=train_x, test_X=test_x)

# add back id and label columns
# add back label columns
transformed_train_data = pd.concat([train_x, train_y.rename(task.label_column)], axis=1)
if task.test_id_column in task.train_data.columns:
if label_column_avaialable_in_test_set:
# Add back label column to test set as it was available before
transformed_test_data = pd.concat([test_x, test_y.rename(task.label_column)], axis=1)
else:
transformed_test_data = test_x

# add back id columns
if task.train_id_column in task.train_data.columns:
transformed_train_data = pd.concat(
[transformed_train_data, task.train_data[task.test_id_column]], axis=1
[transformed_train_data, task.train_data[task.train_id_column]], axis=1
)
transformed_test_data = pd.concat([test_x, task.test_data[task.test_id_column]], axis=1)
if task.test_id_column in task.test_data.columns:
transformed_test_data = pd.concat([transformed_test_data, task.test_data[task.test_id_column]], axis=1)

task = copy.deepcopy(task)
task.train_data = transformed_train_data
Expand Down
Loading