Skip to content

Commit

Permalink
fixed lint issues
Browse files Browse the repository at this point in the history
Signed-off-by: Minura Punchihewa <[email protected]>
  • Loading branch information
MinuraPunchihewa committed Oct 11, 2024
1 parent 0e5a4c4 commit 61f7771
Showing 1 changed file with 7 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,7 @@
from typing import Any

import pandas as pd
import pandas as pd
from kedro.io.core import (
DatasetError
)
from kedro.io.core import DatasetError
from pyspark.sql import DataFrame

from kedro_datasets.databricks._base_table_dataset import BaseTable, BaseTableDataset
Expand All @@ -26,7 +23,7 @@ class ExternalTable(BaseTable):

def _validate_location(self) -> None:
"""Validates that a location is provided if the table does not exist.
Raises:
DatasetError: If the table does not exist and no location is provided.
"""
Expand All @@ -35,10 +32,10 @@ def _validate_location(self) -> None:
"If the external table does not exists, the `location` parameter must be provided. "
"This should be valid path in an external location that has already been created."
)

def _validate_write_mode(self) -> None:
"""Validates that the write mode is compatible with the format.
Raises:
DatasetError: If the write mode is not compatible with the format.
"""
Expand All @@ -49,7 +46,7 @@ def _validate_write_mode(self) -> None:
f"Format '{self.format}' is not supported for upserts. "
f"Please use 'delta' format."
)

if self.write_mode == "overwrite" and self.format != "delta" and not self.location:
raise DatasetError(
f"Format '{self.format}' is supported for overwrites only if the location is provided. "
Expand Down Expand Up @@ -150,7 +147,7 @@ def _create_table( # noqa: PLR0913
primary_key=primary_key,
format=format
)

def _save_overwrite(self, data: DataFrame) -> None:
"""Overwrites the data in the table with the data provided.
Args:
Expand All @@ -159,7 +156,7 @@ def _save_overwrite(self, data: DataFrame) -> None:
writer = data.write.format(self._table.format).mode("overwrite").option(
"overwriteSchema", "true"
)

if self._table.partition_columns:
writer.partitionBy(
*self._table.partition_columns if isinstance(self._table.partition_columns, list) else self._table.partition_columns
Expand Down

0 comments on commit 61f7771

Please sign in to comment.