diff --git a/.gitignore b/.gitignore index b24dc09988..ee243eb41a 100644 --- a/.gitignore +++ b/.gitignore @@ -114,6 +114,7 @@ celerybeat-schedule # Environments .env +.envrc .venv env/ venv/ diff --git a/RELEASE.md b/RELEASE.md index 0f47731608..882174280b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -118,6 +118,7 @@ * Bumped minimum required `fsspec` version to 2021.04. * Fixed the `kedro install` and `kedro build-reqs` flows when uninstalled dependencies are present in a project's `settings.py`, `context.py` or `hooks.py` ([Issue #829](https://github.com/quantumblacklabs/kedro/issues/829)). * Imports are now refactored at `kedro pipeline package` and `kedro pipeline pull` time, so that _aliasing_ a modular pipeline doesn't break it. +* Pinned `dynaconf` to `<3.1.6` because the method signature for `_validate_items` changed which is used in Kedro. ## Minor breaking changes to the API diff --git a/docs/source/05_data/01_data_catalog.md b/docs/source/05_data/01_data_catalog.md index a90c06c5ad..359e847bcc 100644 --- a/docs/source/05_data/01_data_catalog.md +++ b/docs/source/05_data/01_data_catalog.md @@ -361,7 +361,7 @@ CSVDataSet( ## Loading multiple datasets that have similar configuration -You may encounter situations where your datasets use the same file format, load and save arguments, and are stored in the same folder. YAML has a [built-in syntax](https://yaml.org/spec/1.2/#Syntax) for factorising parts of a YAML file, which means that you can decide what is generalisable across your datasets so that you do not have to spend time copying and pasting dataset configurations in `catalog.yml`. +You may encounter situations where your datasets use the same file format, load and save arguments, and are stored in the same folder. YAML has a [built-in syntax](https://yaml.org/spec/1.2.1/#Syntax) for factorising parts of a YAML file, which means that you can decide what is generalisable across your datasets so that you do not have to spend time copying and pasting dataset configurations in `catalog.yml`. You can see this in the following example: diff --git a/kedro/config/__init__.py b/kedro/config/__init__.py index c8493961e5..efec65e597 100644 --- a/kedro/config/__init__.py +++ b/kedro/config/__init__.py @@ -30,9 +30,18 @@ configuration from different file formats. """ +from .abstract_config import ( + AbstractConfigLoader, + BadConfigException, + MissingConfigException, +) +from .config import ConfigLoader +from .templated_config import TemplatedConfigLoader -from .abstract_config import AbstractConfigLoader # NOQA -from .abstract_config import BadConfigException # NOQA -from .abstract_config import MissingConfigException # NOQA -from .config import ConfigLoader # NOQA -from .templated_config import TemplatedConfigLoader # NOQA +__all__ = [ + "AbstractConfigLoader", + "BadConfigException", + "ConfigLoader", + "MissingConfigException", + "TemplatedConfigLoader", +] diff --git a/kedro/extras/datasets/api/__init__.py b/kedro/extras/datasets/api/__init__.py index 205ec40312..87a63e5ad9 100644 --- a/kedro/extras/datasets/api/__init__.py +++ b/kedro/extras/datasets/api/__init__.py @@ -36,4 +36,4 @@ from contextlib import suppress with suppress(ImportError): - from .api_dataset import APIDataSet # NOQA + from .api_dataset import APIDataSet diff --git a/kedro/extras/datasets/biosequence/__init__.py b/kedro/extras/datasets/biosequence/__init__.py index 181335adc4..fdf1811cc7 100644 --- a/kedro/extras/datasets/biosequence/__init__.py +++ b/kedro/extras/datasets/biosequence/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .biosequence_dataset import BioSequenceDataSet # NOQA + from .biosequence_dataset import BioSequenceDataSet diff --git a/kedro/extras/datasets/dask/__init__.py b/kedro/extras/datasets/dask/__init__.py index 895ded5102..79dd06e9b9 100644 --- a/kedro/extras/datasets/dask/__init__.py +++ b/kedro/extras/datasets/dask/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .parquet_dataset import ParquetDataSet # NOQA + from .parquet_dataset import ParquetDataSet diff --git a/kedro/extras/datasets/email/__init__.py b/kedro/extras/datasets/email/__init__.py index c5d1338fdb..6bf351b06d 100644 --- a/kedro/extras/datasets/email/__init__.py +++ b/kedro/extras/datasets/email/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .message_dataset import EmailMessageDataSet # NOQA + from .message_dataset import EmailMessageDataSet diff --git a/kedro/extras/datasets/geopandas/__init__.py b/kedro/extras/datasets/geopandas/__init__.py index b0e7de3b10..dfc2119be8 100644 --- a/kedro/extras/datasets/geopandas/__init__.py +++ b/kedro/extras/datasets/geopandas/__init__.py @@ -32,4 +32,4 @@ from contextlib import suppress with suppress(ImportError): - from .geojson_dataset import GeoJSONDataSet # NOQA + from .geojson_dataset import GeoJSONDataSet diff --git a/kedro/extras/datasets/holoviews/__init__.py b/kedro/extras/datasets/holoviews/__init__.py index 91c50ce1fd..a3c265b7d9 100644 --- a/kedro/extras/datasets/holoviews/__init__.py +++ b/kedro/extras/datasets/holoviews/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .holoviews_writer import HoloviewsWriter # NOQA + from .holoviews_writer import HoloviewsWriter diff --git a/kedro/extras/datasets/json/__init__.py b/kedro/extras/datasets/json/__init__.py index 6e5352ec59..d9e770e6f9 100644 --- a/kedro/extras/datasets/json/__init__.py +++ b/kedro/extras/datasets/json/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .json_dataset import JSONDataSet # NOQA + from .json_dataset import JSONDataSet diff --git a/kedro/extras/datasets/matplotlib/__init__.py b/kedro/extras/datasets/matplotlib/__init__.py index fe2899227d..e955a5caa0 100644 --- a/kedro/extras/datasets/matplotlib/__init__.py +++ b/kedro/extras/datasets/matplotlib/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .matplotlib_writer import MatplotlibWriter # NOQA + from .matplotlib_writer import MatplotlibWriter diff --git a/kedro/extras/datasets/pandas/__init__.py b/kedro/extras/datasets/pandas/__init__.py index 8a7137db38..0561d524ea 100644 --- a/kedro/extras/datasets/pandas/__init__.py +++ b/kedro/extras/datasets/pandas/__init__.py @@ -45,20 +45,20 @@ from contextlib import suppress with suppress(ImportError): - from .csv_dataset import CSVDataSet # NOQA + from .csv_dataset import CSVDataSet with suppress(ImportError): - from .excel_dataset import ExcelDataSet # NOQA + from .excel_dataset import ExcelDataSet with suppress(ImportError): - from .feather_dataset import FeatherDataSet # NOQA + from .feather_dataset import FeatherDataSet with suppress(ImportError): - from .gbq_dataset import GBQTableDataSet # NOQA + from .gbq_dataset import GBQTableDataSet with suppress(ImportError): - from .hdf_dataset import HDFDataSet # NOQA + from .hdf_dataset import HDFDataSet with suppress(ImportError): - from .json_dataset import JSONDataSet # NOQA + from .json_dataset import JSONDataSet with suppress(ImportError): - from .parquet_dataset import ParquetDataSet # NOQA + from .parquet_dataset import ParquetDataSet with suppress(ImportError): - from .sql_dataset import SQLQueryDataSet, SQLTableDataSet # NOQA + from .sql_dataset import SQLQueryDataSet, SQLTableDataSet with suppress(ImportError): - from .xml_dataset import XMLDataSet # NOQA + from .xml_dataset import XMLDataSet diff --git a/kedro/extras/datasets/pickle/__init__.py b/kedro/extras/datasets/pickle/__init__.py index b876689bd0..cef97483e0 100644 --- a/kedro/extras/datasets/pickle/__init__.py +++ b/kedro/extras/datasets/pickle/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .pickle_dataset import PickleDataSet # NOQA + from .pickle_dataset import PickleDataSet diff --git a/kedro/extras/datasets/pillow/__init__.py b/kedro/extras/datasets/pillow/__init__.py index 1426ec8290..8d97342db3 100644 --- a/kedro/extras/datasets/pillow/__init__.py +++ b/kedro/extras/datasets/pillow/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .image_dataset import ImageDataSet # NOQA + from .image_dataset import ImageDataSet diff --git a/kedro/extras/datasets/plotly/__init__.py b/kedro/extras/datasets/plotly/__init__.py index 229220e0a6..3bc63e483b 100644 --- a/kedro/extras/datasets/plotly/__init__.py +++ b/kedro/extras/datasets/plotly/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .plotly_dataset import PlotlyDataSet # NOQA + from .plotly_dataset import PlotlyDataSet diff --git a/kedro/extras/datasets/spark/__init__.py b/kedro/extras/datasets/spark/__init__.py index fc637663cd..87dcf30e84 100644 --- a/kedro/extras/datasets/spark/__init__.py +++ b/kedro/extras/datasets/spark/__init__.py @@ -33,8 +33,8 @@ from contextlib import suppress with suppress(ImportError): - from .spark_dataset import SparkDataSet # NOQA + from .spark_dataset import SparkDataSet with suppress(ImportError): - from .spark_hive_dataset import SparkHiveDataSet # NOQA + from .spark_hive_dataset import SparkHiveDataSet with suppress(ImportError): - from .spark_jdbc_dataset import SparkJDBCDataSet # NOQA + from .spark_jdbc_dataset import SparkJDBCDataSet diff --git a/kedro/extras/datasets/tensorflow/__init__.py b/kedro/extras/datasets/tensorflow/__init__.py index 8fac25a8bd..85942f949c 100644 --- a/kedro/extras/datasets/tensorflow/__init__.py +++ b/kedro/extras/datasets/tensorflow/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .tensorflow_model_dataset import TensorFlowModelDataset # NOQA + from .tensorflow_model_dataset import TensorFlowModelDataset diff --git a/kedro/extras/datasets/text/__init__.py b/kedro/extras/datasets/text/__init__.py index 5c0618cbfc..dba734fd9e 100644 --- a/kedro/extras/datasets/text/__init__.py +++ b/kedro/extras/datasets/text/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .text_dataset import TextDataSet # NOQA + from .text_dataset import TextDataSet diff --git a/kedro/extras/datasets/tracking/__init__.py b/kedro/extras/datasets/tracking/__init__.py index 62e067029a..c726649223 100644 --- a/kedro/extras/datasets/tracking/__init__.py +++ b/kedro/extras/datasets/tracking/__init__.py @@ -34,6 +34,6 @@ from contextlib import suppress with suppress(ImportError): - from kedro.extras.datasets.tracking.metrics_dataset import MetricsDataSet # NOQA + from kedro.extras.datasets.tracking.metrics_dataset import MetricsDataSet with suppress(ImportError): - from kedro.extras.datasets.tracking.json_dataset import JSONDataSet # NOQA + from kedro.extras.datasets.tracking.json_dataset import JSONDataSet diff --git a/kedro/extras/datasets/yaml/__init__.py b/kedro/extras/datasets/yaml/__init__.py index b443fd271b..c18b5751ec 100644 --- a/kedro/extras/datasets/yaml/__init__.py +++ b/kedro/extras/datasets/yaml/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .yaml_dataset import YAMLDataSet # NOQA + from .yaml_dataset import YAMLDataSet diff --git a/kedro/extras/logging/__init__.py b/kedro/extras/logging/__init__.py index 337060fb10..16caef6945 100644 --- a/kedro/extras/logging/__init__.py +++ b/kedro/extras/logging/__init__.py @@ -30,4 +30,6 @@ This module contains a logging handler class which produces coloured logs. """ -from .color_logger import ColorHandler # NOQA +from .color_logger import ColorHandler + +__all__ = ["ColorHandler"] diff --git a/kedro/extras/transformers/__init__.py b/kedro/extras/transformers/__init__.py index 302136e01f..5a9b3e9e21 100644 --- a/kedro/extras/transformers/__init__.py +++ b/kedro/extras/transformers/__init__.py @@ -28,5 +28,7 @@ """``kedro.extras.transformers`` is the home of Kedro's dataset transformers.""" -from .memory_profiler import ProfileMemoryTransformer # NOQA -from .time_profiler import ProfileTimeTransformer # NOQA +from .memory_profiler import ProfileMemoryTransformer +from .time_profiler import ProfileTimeTransformer + +__all__ = ["ProfileMemoryTransformer", "ProfileTimeTransformer"] diff --git a/kedro/framework/cli/__init__.py b/kedro/framework/cli/__init__.py index ef812fc25b..b46675be9a 100644 --- a/kedro/framework/cli/__init__.py +++ b/kedro/framework/cli/__init__.py @@ -29,5 +29,7 @@ """``kedro.framework.cli`` implements commands available from Kedro's CLI. """ -from .cli import main # NOQA -from .utils import command_with_verbosity, load_entry_points # NOQA +from .cli import main +from .utils import command_with_verbosity, load_entry_points + +__all__ = ["main", "command_with_verbosity", "load_entry_points"] diff --git a/kedro/framework/cli/hooks/__init__.py b/kedro/framework/cli/hooks/__init__.py index bb6a1a2a98..c07e40a3ef 100644 --- a/kedro/framework/cli/hooks/__init__.py +++ b/kedro/framework/cli/hooks/__init__.py @@ -26,5 +26,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """``kedro.framework.cli.hooks`` provides primitives to use hooks to extend KedroCLI's behaviour""" -from .manager import CLIHooksManager # NOQA -from .markers import cli_hook_impl # NOQA +from .manager import CLIHooksManager +from .markers import cli_hook_impl + +__all__ = ["CLIHooksManager", "cli_hook_impl"] diff --git a/kedro/framework/context/__init__.py b/kedro/framework/context/__init__.py index f5288f38fa..bb3aa23972 100644 --- a/kedro/framework/context/__init__.py +++ b/kedro/framework/context/__init__.py @@ -30,5 +30,6 @@ project context. """ -from .context import KedroContext # NOQA -from .context import KedroContextError # NOQA +from .context import KedroContext, KedroContextError + +__all__ = ["KedroContext", "KedroContextError"] diff --git a/kedro/framework/hooks/__init__.py b/kedro/framework/hooks/__init__.py index f0fcf4aaba..6d6c03c830 100644 --- a/kedro/framework/hooks/__init__.py +++ b/kedro/framework/hooks/__init__.py @@ -26,5 +26,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """``kedro.framework.hooks`` provides primitives to use hooks to extend KedroContext's behaviour""" -from .manager import get_hook_manager # NOQA -from .markers import hook_impl # NOQA +from .manager import get_hook_manager +from .markers import hook_impl + +__all__ = ["get_hook_manager", "hook_impl"] diff --git a/kedro/framework/session/__init__.py b/kedro/framework/session/__init__.py index d740abdb73..6037aff1ad 100644 --- a/kedro/framework/session/__init__.py +++ b/kedro/framework/session/__init__.py @@ -29,4 +29,6 @@ """``kedro.framework.session`` provides access to KedroSession responsible for project lifecycle. """ -from .session import KedroSession, get_current_session # NOQA +from .session import KedroSession, get_current_session + +__all__ = ["KedroSession", "get_current_session"] diff --git a/kedro/io/__init__.py b/kedro/io/__init__.py index d85264bde3..d5c0ee6bfb 100644 --- a/kedro/io/__init__.py +++ b/kedro/io/__init__.py @@ -31,17 +31,35 @@ which allows implementation of various ``AbstractDataSet``s. """ -from .cached_dataset import CachedDataSet # NOQA -from .core import AbstractDataSet # NOQA -from .core import AbstractVersionedDataSet # NOQA -from .core import DataSetAlreadyExistsError # NOQA -from .core import DataSetError # NOQA -from .core import DataSetNotFoundError # NOQA -from .core import Version # NOQA -from .data_catalog import DataCatalog # NOQA -from .data_catalog_with_default import DataCatalogWithDefault # NOQA -from .lambda_dataset import LambdaDataSet # NOQA -from .memory_dataset import MemoryDataSet # NOQA -from .partitioned_dataset import IncrementalDataSet # NOQA -from .partitioned_dataset import PartitionedDataSet # NOQA -from .transformers import AbstractTransformer # NOQA +from .cached_dataset import CachedDataSet +from .core import ( + AbstractDataSet, + AbstractVersionedDataSet, + DataSetAlreadyExistsError, + DataSetError, + DataSetNotFoundError, + Version, +) +from .data_catalog import DataCatalog +from .data_catalog_with_default import DataCatalogWithDefault +from .lambda_dataset import LambdaDataSet +from .memory_dataset import MemoryDataSet +from .partitioned_dataset import IncrementalDataSet, PartitionedDataSet +from .transformers import AbstractTransformer + +__all__ = [ + "AbstractDataSet", + "AbstractTransformer", + "AbstractVersionedDataSet", + "CachedDataSet", + "DataCatalog", + "DataCatalogWithDefault", + "DataSetAlreadyExistsError", + "DataSetError", + "DataSetNotFoundError", + "IncrementalDataSet", + "LambdaDataSet", + "MemoryDataSet", + "PartitionedDataSet", + "Version", +] diff --git a/kedro/pipeline/__init__.py b/kedro/pipeline/__init__.py index 069d17e7c8..595a8ddecb 100644 --- a/kedro/pipeline/__init__.py +++ b/kedro/pipeline/__init__.py @@ -30,6 +30,8 @@ data-driven pipelines. """ -from .modular_pipeline import pipeline # NOQA -from .node import node # NOQA -from .pipeline import Pipeline # NOQA +from .modular_pipeline import pipeline +from .node import node +from .pipeline import Pipeline + +__all__ = ["pipeline", "node", "Pipeline"] diff --git a/kedro/runner/__init__.py b/kedro/runner/__init__.py index 5d06a7ddeb..7da87932b7 100644 --- a/kedro/runner/__init__.py +++ b/kedro/runner/__init__.py @@ -30,7 +30,15 @@ to execute ``Pipeline`` instances. """ -from .parallel_runner import ParallelRunner # NOQA -from .runner import AbstractRunner, run_node # NOQA -from .sequential_runner import SequentialRunner # NOQA -from .thread_runner import ThreadRunner # NOQA +from .parallel_runner import ParallelRunner +from .runner import AbstractRunner, run_node +from .sequential_runner import SequentialRunner +from .thread_runner import ThreadRunner + +__all__ = [ + "AbstractRunner", + "ParallelRunner", + "SequentialRunner", + "ThreadRunner", + "run_node", +] diff --git a/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py b/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py index 4a2ff49991..865d2e8e80 100644 --- a/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py +++ b/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py @@ -30,6 +30,8 @@ generated using Kedro {{ cookiecutter.kedro_version }} """ -from .pipeline import create_pipeline # NOQA +from .pipeline import create_pipeline + +__all__ = ["create_pipeline"] __version__ = "0.1" diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/.gitignore b/kedro/templates/project/{{ cookiecutter.repo_name }}/.gitignore index 1f0b415c17..8cd10b4acc 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/.gitignore +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/.gitignore @@ -144,6 +144,7 @@ celerybeat-schedule # Environments .env +.envrc .venv env/ venv/ diff --git a/kedro/versioning/__init__.py b/kedro/versioning/__init__.py index e8fd72089c..2e419b80ee 100644 --- a/kedro/versioning/__init__.py +++ b/kedro/versioning/__init__.py @@ -30,4 +30,6 @@ capturing information required to reproduce a Kedro run. """ -from .journal import Journal # NOQA +from .journal import Journal + +__all__ = ["Journal"] diff --git a/requirements.txt b/requirements.txt index 5ba6ff036a..2f8a201cf0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ anyconfig~=0.10.0 cachetools~=4.1 click<8.0 cookiecutter~=1.7.0 -dynaconf~=3.1.2 +dynaconf<3.1.6 # Pinned because Dynaconf broke a method signature in 3.1.6 used in Kedro fsspec>=2021.04, <2022.01 # Upper bound set arbitrarily, to be reassessed in early 2022 gitpython~=3.0 jmespath>=0.9.5, <1.0