From ef32bfd85515f75cf204f203cc3f2a0a0cc75728 Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Thu, 9 Sep 2021 04:12:41 -0500 Subject: [PATCH 1/5] implement __all__ in __init__ modules (#874) --- kedro/config/__init__.py | 13 ++++-- kedro/extras/datasets/api/__init__.py | 2 +- kedro/extras/datasets/biosequence/__init__.py | 2 +- kedro/extras/datasets/dask/__init__.py | 2 +- kedro/extras/datasets/email/__init__.py | 2 +- kedro/extras/datasets/geopandas/__init__.py | 2 +- kedro/extras/datasets/holoviews/__init__.py | 2 +- kedro/extras/datasets/json/__init__.py | 2 +- kedro/extras/datasets/matplotlib/__init__.py | 2 +- kedro/extras/datasets/networkx/__init__.py | 2 +- kedro/extras/datasets/pandas/__init__.py | 18 ++++---- kedro/extras/datasets/pickle/__init__.py | 2 +- kedro/extras/datasets/pillow/__init__.py | 2 +- kedro/extras/datasets/plotly/__init__.py | 2 +- kedro/extras/datasets/spark/__init__.py | 6 +-- kedro/extras/datasets/tensorflow/__init__.py | 2 +- kedro/extras/datasets/text/__init__.py | 2 +- kedro/extras/datasets/tracking/__init__.py | 4 +- kedro/extras/datasets/yaml/__init__.py | 2 +- kedro/extras/logging/__init__.py | 4 +- kedro/extras/transformers/__init__.py | 6 ++- kedro/framework/cli/__init__.py | 6 ++- kedro/framework/cli/hooks/__init__.py | 6 ++- kedro/framework/context/__init__.py | 6 +-- kedro/framework/hooks/__init__.py | 6 ++- kedro/framework/session/__init__.py | 4 +- kedro/io/__init__.py | 46 +++++++++++++------ kedro/pipeline/__init__.py | 8 ++-- kedro/runner/__init__.py | 16 +++++-- .../__init__.py | 4 +- kedro/versioning/__init__.py | 4 +- 31 files changed, 118 insertions(+), 69 deletions(-) diff --git a/kedro/config/__init__.py b/kedro/config/__init__.py index 4bce25b0d6..2dc1f64c31 100644 --- a/kedro/config/__init__.py +++ b/kedro/config/__init__.py @@ -31,7 +31,12 @@ """ -from .config import BadConfigException # NOQA -from .config import ConfigLoader # NOQA -from .config import MissingConfigException # NOQA -from .templated_config import TemplatedConfigLoader # NOQA +from .config import BadConfigException, ConfigLoader, MissingConfigException +from .templated_config import TemplatedConfigLoader + +__all__ = [ + "BadConfigException", + "ConfigLoader", + "MissingConfigException", + "TemplatedConfigLoader", +] diff --git a/kedro/extras/datasets/api/__init__.py b/kedro/extras/datasets/api/__init__.py index 205ec40312..87a63e5ad9 100644 --- a/kedro/extras/datasets/api/__init__.py +++ b/kedro/extras/datasets/api/__init__.py @@ -36,4 +36,4 @@ from contextlib import suppress with suppress(ImportError): - from .api_dataset import APIDataSet # NOQA + from .api_dataset import APIDataSet diff --git a/kedro/extras/datasets/biosequence/__init__.py b/kedro/extras/datasets/biosequence/__init__.py index 181335adc4..fdf1811cc7 100644 --- a/kedro/extras/datasets/biosequence/__init__.py +++ b/kedro/extras/datasets/biosequence/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .biosequence_dataset import BioSequenceDataSet # NOQA + from .biosequence_dataset import BioSequenceDataSet diff --git a/kedro/extras/datasets/dask/__init__.py b/kedro/extras/datasets/dask/__init__.py index 895ded5102..79dd06e9b9 100644 --- a/kedro/extras/datasets/dask/__init__.py +++ b/kedro/extras/datasets/dask/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .parquet_dataset import ParquetDataSet # NOQA + from .parquet_dataset import ParquetDataSet diff --git a/kedro/extras/datasets/email/__init__.py b/kedro/extras/datasets/email/__init__.py index c5d1338fdb..6bf351b06d 100644 --- a/kedro/extras/datasets/email/__init__.py +++ b/kedro/extras/datasets/email/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .message_dataset import EmailMessageDataSet # NOQA + from .message_dataset import EmailMessageDataSet diff --git a/kedro/extras/datasets/geopandas/__init__.py b/kedro/extras/datasets/geopandas/__init__.py index b0e7de3b10..dfc2119be8 100644 --- a/kedro/extras/datasets/geopandas/__init__.py +++ b/kedro/extras/datasets/geopandas/__init__.py @@ -32,4 +32,4 @@ from contextlib import suppress with suppress(ImportError): - from .geojson_dataset import GeoJSONDataSet # NOQA + from .geojson_dataset import GeoJSONDataSet diff --git a/kedro/extras/datasets/holoviews/__init__.py b/kedro/extras/datasets/holoviews/__init__.py index 91c50ce1fd..a3c265b7d9 100644 --- a/kedro/extras/datasets/holoviews/__init__.py +++ b/kedro/extras/datasets/holoviews/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .holoviews_writer import HoloviewsWriter # NOQA + from .holoviews_writer import HoloviewsWriter diff --git a/kedro/extras/datasets/json/__init__.py b/kedro/extras/datasets/json/__init__.py index 6e5352ec59..d9e770e6f9 100644 --- a/kedro/extras/datasets/json/__init__.py +++ b/kedro/extras/datasets/json/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .json_dataset import JSONDataSet # NOQA + from .json_dataset import JSONDataSet diff --git a/kedro/extras/datasets/matplotlib/__init__.py b/kedro/extras/datasets/matplotlib/__init__.py index fe2899227d..e955a5caa0 100644 --- a/kedro/extras/datasets/matplotlib/__init__.py +++ b/kedro/extras/datasets/matplotlib/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .matplotlib_writer import MatplotlibWriter # NOQA + from .matplotlib_writer import MatplotlibWriter diff --git a/kedro/extras/datasets/networkx/__init__.py b/kedro/extras/datasets/networkx/__init__.py index 377692cc6c..a5574150c8 100644 --- a/kedro/extras/datasets/networkx/__init__.py +++ b/kedro/extras/datasets/networkx/__init__.py @@ -34,4 +34,4 @@ from contextlib import suppress with suppress(ImportError): - from .networkx_dataset import NetworkXDataSet # NOQA + from .networkx_dataset import NetworkXDataSet diff --git a/kedro/extras/datasets/pandas/__init__.py b/kedro/extras/datasets/pandas/__init__.py index 3f26bf00b1..b769e8357b 100644 --- a/kedro/extras/datasets/pandas/__init__.py +++ b/kedro/extras/datasets/pandas/__init__.py @@ -45,20 +45,20 @@ from contextlib import suppress with suppress(ImportError): - from .csv_dataset import CSVDataSet # NOQA + from .csv_dataset import CSVDataSet with suppress(ImportError): - from .excel_dataset import ExcelDataSet # NOQA + from .excel_dataset import ExcelDataSet with suppress(ImportError): - from .appendable_excel_dataset import AppendableExcelDataSet # NOQA + from .appendable_excel_dataset import AppendableExcelDataSet with suppress(ImportError): - from .feather_dataset import FeatherDataSet # NOQA + from .feather_dataset import FeatherDataSet with suppress(ImportError): - from .gbq_dataset import GBQTableDataSet # NOQA + from .gbq_dataset import GBQTableDataSet with suppress(ImportError): - from .hdf_dataset import HDFDataSet # NOQA + from .hdf_dataset import HDFDataSet with suppress(ImportError): - from .json_dataset import JSONDataSet # NOQA + from .json_dataset import JSONDataSet with suppress(ImportError): - from .parquet_dataset import ParquetDataSet # NOQA + from .parquet_dataset import ParquetDataSet with suppress(ImportError): - from .sql_dataset import SQLQueryDataSet, SQLTableDataSet # NOQA + from .sql_dataset import SQLQueryDataSet, SQLTableDataSet diff --git a/kedro/extras/datasets/pickle/__init__.py b/kedro/extras/datasets/pickle/__init__.py index b876689bd0..cef97483e0 100644 --- a/kedro/extras/datasets/pickle/__init__.py +++ b/kedro/extras/datasets/pickle/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .pickle_dataset import PickleDataSet # NOQA + from .pickle_dataset import PickleDataSet diff --git a/kedro/extras/datasets/pillow/__init__.py b/kedro/extras/datasets/pillow/__init__.py index 1426ec8290..8d97342db3 100644 --- a/kedro/extras/datasets/pillow/__init__.py +++ b/kedro/extras/datasets/pillow/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .image_dataset import ImageDataSet # NOQA + from .image_dataset import ImageDataSet diff --git a/kedro/extras/datasets/plotly/__init__.py b/kedro/extras/datasets/plotly/__init__.py index 229220e0a6..3bc63e483b 100644 --- a/kedro/extras/datasets/plotly/__init__.py +++ b/kedro/extras/datasets/plotly/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .plotly_dataset import PlotlyDataSet # NOQA + from .plotly_dataset import PlotlyDataSet diff --git a/kedro/extras/datasets/spark/__init__.py b/kedro/extras/datasets/spark/__init__.py index fc637663cd..87dcf30e84 100644 --- a/kedro/extras/datasets/spark/__init__.py +++ b/kedro/extras/datasets/spark/__init__.py @@ -33,8 +33,8 @@ from contextlib import suppress with suppress(ImportError): - from .spark_dataset import SparkDataSet # NOQA + from .spark_dataset import SparkDataSet with suppress(ImportError): - from .spark_hive_dataset import SparkHiveDataSet # NOQA + from .spark_hive_dataset import SparkHiveDataSet with suppress(ImportError): - from .spark_jdbc_dataset import SparkJDBCDataSet # NOQA + from .spark_jdbc_dataset import SparkJDBCDataSet diff --git a/kedro/extras/datasets/tensorflow/__init__.py b/kedro/extras/datasets/tensorflow/__init__.py index 8fac25a8bd..85942f949c 100644 --- a/kedro/extras/datasets/tensorflow/__init__.py +++ b/kedro/extras/datasets/tensorflow/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .tensorflow_model_dataset import TensorFlowModelDataset # NOQA + from .tensorflow_model_dataset import TensorFlowModelDataset diff --git a/kedro/extras/datasets/text/__init__.py b/kedro/extras/datasets/text/__init__.py index 5c0618cbfc..dba734fd9e 100644 --- a/kedro/extras/datasets/text/__init__.py +++ b/kedro/extras/datasets/text/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .text_dataset import TextDataSet # NOQA + from .text_dataset import TextDataSet diff --git a/kedro/extras/datasets/tracking/__init__.py b/kedro/extras/datasets/tracking/__init__.py index 62e067029a..c726649223 100644 --- a/kedro/extras/datasets/tracking/__init__.py +++ b/kedro/extras/datasets/tracking/__init__.py @@ -34,6 +34,6 @@ from contextlib import suppress with suppress(ImportError): - from kedro.extras.datasets.tracking.metrics_dataset import MetricsDataSet # NOQA + from kedro.extras.datasets.tracking.metrics_dataset import MetricsDataSet with suppress(ImportError): - from kedro.extras.datasets.tracking.json_dataset import JSONDataSet # NOQA + from kedro.extras.datasets.tracking.json_dataset import JSONDataSet diff --git a/kedro/extras/datasets/yaml/__init__.py b/kedro/extras/datasets/yaml/__init__.py index b443fd271b..c18b5751ec 100644 --- a/kedro/extras/datasets/yaml/__init__.py +++ b/kedro/extras/datasets/yaml/__init__.py @@ -33,4 +33,4 @@ from contextlib import suppress with suppress(ImportError): - from .yaml_dataset import YAMLDataSet # NOQA + from .yaml_dataset import YAMLDataSet diff --git a/kedro/extras/logging/__init__.py b/kedro/extras/logging/__init__.py index 337060fb10..16caef6945 100644 --- a/kedro/extras/logging/__init__.py +++ b/kedro/extras/logging/__init__.py @@ -30,4 +30,6 @@ This module contains a logging handler class which produces coloured logs. """ -from .color_logger import ColorHandler # NOQA +from .color_logger import ColorHandler + +__all__ = ["ColorHandler"] diff --git a/kedro/extras/transformers/__init__.py b/kedro/extras/transformers/__init__.py index 302136e01f..5a9b3e9e21 100644 --- a/kedro/extras/transformers/__init__.py +++ b/kedro/extras/transformers/__init__.py @@ -28,5 +28,7 @@ """``kedro.extras.transformers`` is the home of Kedro's dataset transformers.""" -from .memory_profiler import ProfileMemoryTransformer # NOQA -from .time_profiler import ProfileTimeTransformer # NOQA +from .memory_profiler import ProfileMemoryTransformer +from .time_profiler import ProfileTimeTransformer + +__all__ = ["ProfileMemoryTransformer", "ProfileTimeTransformer"] diff --git a/kedro/framework/cli/__init__.py b/kedro/framework/cli/__init__.py index 8684322a7a..9153e635f3 100644 --- a/kedro/framework/cli/__init__.py +++ b/kedro/framework/cli/__init__.py @@ -29,5 +29,7 @@ """``kedro.framework.cli`` implements commands available from Kedro's CLI. """ -from .cli import get_project_context, main # NOQA -from .utils import command_with_verbosity, load_entry_points # NOQA +from .cli import get_project_context, main +from .utils import command_with_verbosity, load_entry_points + +__all__ = ["get_project_context", "main", "command_with_verbosity", "load_entry_points"] diff --git a/kedro/framework/cli/hooks/__init__.py b/kedro/framework/cli/hooks/__init__.py index bb6a1a2a98..c07e40a3ef 100644 --- a/kedro/framework/cli/hooks/__init__.py +++ b/kedro/framework/cli/hooks/__init__.py @@ -26,5 +26,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """``kedro.framework.cli.hooks`` provides primitives to use hooks to extend KedroCLI's behaviour""" -from .manager import CLIHooksManager # NOQA -from .markers import cli_hook_impl # NOQA +from .manager import CLIHooksManager +from .markers import cli_hook_impl + +__all__ = ["CLIHooksManager", "cli_hook_impl"] diff --git a/kedro/framework/context/__init__.py b/kedro/framework/context/__init__.py index fae6342a1b..be734e3b3d 100644 --- a/kedro/framework/context/__init__.py +++ b/kedro/framework/context/__init__.py @@ -30,6 +30,6 @@ project context. """ -from .context import KedroContext # NOQA -from .context import KedroContextError # NOQA -from .context import load_context # NOQA +from .context import KedroContext, KedroContextError, load_context + +__all__ = ["KedroContext", "KedroContextError", "load_context"] diff --git a/kedro/framework/hooks/__init__.py b/kedro/framework/hooks/__init__.py index f0fcf4aaba..6d6c03c830 100644 --- a/kedro/framework/hooks/__init__.py +++ b/kedro/framework/hooks/__init__.py @@ -26,5 +26,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """``kedro.framework.hooks`` provides primitives to use hooks to extend KedroContext's behaviour""" -from .manager import get_hook_manager # NOQA -from .markers import hook_impl # NOQA +from .manager import get_hook_manager +from .markers import hook_impl + +__all__ = ["get_hook_manager", "hook_impl"] diff --git a/kedro/framework/session/__init__.py b/kedro/framework/session/__init__.py index d740abdb73..6037aff1ad 100644 --- a/kedro/framework/session/__init__.py +++ b/kedro/framework/session/__init__.py @@ -29,4 +29,6 @@ """``kedro.framework.session`` provides access to KedroSession responsible for project lifecycle. """ -from .session import KedroSession, get_current_session # NOQA +from .session import KedroSession, get_current_session + +__all__ = ["KedroSession", "get_current_session"] diff --git a/kedro/io/__init__.py b/kedro/io/__init__.py index be3c51f381..dde54f72a8 100644 --- a/kedro/io/__init__.py +++ b/kedro/io/__init__.py @@ -31,17 +31,35 @@ which allows implementation of various ``AbstractDataSet``s. """ -from .cached_dataset import CachedDataSet # NOQA -from .core import AbstractDataSet # NOQA -from .core import AbstractVersionedDataSet # NOQA -from .core import DataSetAlreadyExistsError # NOQA -from .core import DataSetError # NOQA -from .core import DataSetNotFoundError # NOQA -from .core import Version # NOQA -from .data_catalog import DataCatalog # NOQA -from .data_catalog_with_default import DataCatalogWithDefault # NOQA -from .lambda_data_set import LambdaDataSet # NOQA -from .memory_data_set import MemoryDataSet # NOQA -from .partitioned_data_set import IncrementalDataSet # NOQA -from .partitioned_data_set import PartitionedDataSet # NOQA -from .transformers import AbstractTransformer # NOQA +from .cached_dataset import CachedDataSet +from .core import ( + AbstractDataSet, + AbstractVersionedDataSet, + DataSetAlreadyExistsError, + DataSetError, + DataSetNotFoundError, + Version, +) +from .data_catalog import DataCatalog +from .data_catalog_with_default import DataCatalogWithDefault +from .lambda_data_set import LambdaDataSet +from .memory_data_set import MemoryDataSet +from .partitioned_data_set import IncrementalDataSet, PartitionedDataSet +from .transformers import AbstractTransformer + +__all__ = [ + "AbstractDataSet", + "AbstractTransformer", + "AbstractVersionedDataSet", + "CachedDataSet", + "DataCatalog", + "DataCatalogWithDefault", + "DataSetAlreadyExistsError", + "DataSetError", + "DataSetNotFoundError", + "IncrementalDataSet", + "LambdaDataSet", + "MemoryDataSet", + "PartitionedDataSet", + "Version", +] diff --git a/kedro/pipeline/__init__.py b/kedro/pipeline/__init__.py index 069d17e7c8..595a8ddecb 100644 --- a/kedro/pipeline/__init__.py +++ b/kedro/pipeline/__init__.py @@ -30,6 +30,8 @@ data-driven pipelines. """ -from .modular_pipeline import pipeline # NOQA -from .node import node # NOQA -from .pipeline import Pipeline # NOQA +from .modular_pipeline import pipeline +from .node import node +from .pipeline import Pipeline + +__all__ = ["pipeline", "node", "Pipeline"] diff --git a/kedro/runner/__init__.py b/kedro/runner/__init__.py index 5d06a7ddeb..7da87932b7 100644 --- a/kedro/runner/__init__.py +++ b/kedro/runner/__init__.py @@ -30,7 +30,15 @@ to execute ``Pipeline`` instances. """ -from .parallel_runner import ParallelRunner # NOQA -from .runner import AbstractRunner, run_node # NOQA -from .sequential_runner import SequentialRunner # NOQA -from .thread_runner import ThreadRunner # NOQA +from .parallel_runner import ParallelRunner +from .runner import AbstractRunner, run_node +from .sequential_runner import SequentialRunner +from .thread_runner import ThreadRunner + +__all__ = [ + "AbstractRunner", + "ParallelRunner", + "SequentialRunner", + "ThreadRunner", + "run_node", +] diff --git a/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py b/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py index 848c35168d..7220dd299c 100644 --- a/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py +++ b/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py @@ -30,4 +30,6 @@ generated using Kedro {{ cookiecutter.kedro_version }} """ -from .pipeline import create_pipeline # NOQA +from .pipeline import create_pipeline + +__all__ = ["create_pipeline"] diff --git a/kedro/versioning/__init__.py b/kedro/versioning/__init__.py index e8fd72089c..2e419b80ee 100644 --- a/kedro/versioning/__init__.py +++ b/kedro/versioning/__init__.py @@ -30,4 +30,6 @@ capturing information required to reproduce a Kedro run. """ -from .journal import Journal # NOQA +from .journal import Journal + +__all__ = ["Journal"] From 2e3a20a75958735bdba57fc1056742cb3f9b8c20 Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Thu, 9 Sep 2021 04:33:59 -0500 Subject: [PATCH 2/5] Ignore .envrc for framework (#872) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9924f5087a..e99cbc2524 100644 --- a/.gitignore +++ b/.gitignore @@ -114,6 +114,7 @@ celerybeat-schedule # Environments .env +.envrc .venv env/ venv/ From 05542fb483b978cf3e8dda02fafdf32eab4c6a10 Mon Sep 17 00:00:00 2001 From: Waylon Walker Date: Thu, 9 Sep 2021 07:59:35 -0500 Subject: [PATCH 3/5] Ignore .envrc in kedro projects (#873) --- kedro/templates/project/{{ cookiecutter.repo_name }}/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/.gitignore b/kedro/templates/project/{{ cookiecutter.repo_name }}/.gitignore index 1f0b415c17..8cd10b4acc 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/.gitignore +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/.gitignore @@ -144,6 +144,7 @@ celerybeat-schedule # Environments .env +.envrc .venv env/ venv/ From d36fce90f23a79990ea6470ad2ef9b3f1a4779bf Mon Sep 17 00:00:00 2001 From: Merel Theisen <49397448+MerelTheisenQB@users.noreply.github.com> Date: Mon, 13 Sep 2021 15:20:19 +0100 Subject: [PATCH 4/5] [KED-2840] Pin dynaconf to 3.1.5 (#1237) --- RELEASE.md | 1 + docs/source/05_data/01_data_catalog.md | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 28a89a2a86..c424c7e375 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -20,6 +20,7 @@ * Bumped minimum required `fsspec` version to 2021.04. * Fixed the `kedro install` and `kedro build-reqs` flows when uninstalled dependencies are present in a project's `settings.py`, `context.py` or `hooks.py` ([Issue #829](https://github.com/quantumblacklabs/kedro/issues/829)). * Imports are now refactored at `kedro pipeline package` and `kedro pipeline pull` time, so that _aliasing_ a modular pipeline doesn't break it. +* Pinned `dynaconf` to `<3.1.6` because the method signature for `_validate_items` changed which is used in Kedro. ## Minor breaking changes to the API diff --git a/docs/source/05_data/01_data_catalog.md b/docs/source/05_data/01_data_catalog.md index a90c06c5ad..359e847bcc 100644 --- a/docs/source/05_data/01_data_catalog.md +++ b/docs/source/05_data/01_data_catalog.md @@ -361,7 +361,7 @@ CSVDataSet( ## Loading multiple datasets that have similar configuration -You may encounter situations where your datasets use the same file format, load and save arguments, and are stored in the same folder. YAML has a [built-in syntax](https://yaml.org/spec/1.2/#Syntax) for factorising parts of a YAML file, which means that you can decide what is generalisable across your datasets so that you do not have to spend time copying and pasting dataset configurations in `catalog.yml`. +You may encounter situations where your datasets use the same file format, load and save arguments, and are stored in the same folder. YAML has a [built-in syntax](https://yaml.org/spec/1.2.1/#Syntax) for factorising parts of a YAML file, which means that you can decide what is generalisable across your datasets so that you do not have to spend time copying and pasting dataset configurations in `catalog.yml`. You can see this in the following example: diff --git a/requirements.txt b/requirements.txt index 5ba6ff036a..2f8a201cf0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ anyconfig~=0.10.0 cachetools~=4.1 click<8.0 cookiecutter~=1.7.0 -dynaconf~=3.1.2 +dynaconf<3.1.6 # Pinned because Dynaconf broke a method signature in 3.1.6 used in Kedro fsspec>=2021.04, <2022.01 # Upper bound set arbitrarily, to be reassessed in early 2022 gitpython~=3.0 jmespath>=0.9.5, <1.0 From ed594b9d9c0978decdb7f962beae871a79043fa8 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Tue, 14 Sep 2021 11:48:07 +0100 Subject: [PATCH 5/5] Fix merge mistakes --- kedro/config/__init__.py | 8 ++++++-- kedro/extras/datasets/networkx/__init__.py | 2 +- kedro/extras/datasets/pandas/__init__.py | 2 -- kedro/framework/cli/__init__.py | 2 +- kedro/framework/context/__init__.py | 4 ++-- kedro/io/__init__.py | 8 ++++---- .../pipeline/{{ cookiecutter.pipeline_name }}/__init__.py | 2 +- 7 files changed, 15 insertions(+), 13 deletions(-) diff --git a/kedro/config/__init__.py b/kedro/config/__init__.py index 25473758c2..efec65e597 100644 --- a/kedro/config/__init__.py +++ b/kedro/config/__init__.py @@ -30,7 +30,11 @@ configuration from different file formats. """ -from .abstract_config import AbstractConfigLoader, BadConfigException, MissingConfigException +from .abstract_config import ( + AbstractConfigLoader, + BadConfigException, + MissingConfigException, +) from .config import ConfigLoader from .templated_config import TemplatedConfigLoader @@ -40,4 +44,4 @@ "ConfigLoader", "MissingConfigException", "TemplatedConfigLoader", -] \ No newline at end of file +] diff --git a/kedro/extras/datasets/networkx/__init__.py b/kedro/extras/datasets/networkx/__init__.py index 6752e7ad4b..c7c312bf1e 100644 --- a/kedro/extras/datasets/networkx/__init__.py +++ b/kedro/extras/datasets/networkx/__init__.py @@ -40,4 +40,4 @@ from .graphml_dataset import GraphMLDataSet with suppress(ImportError): - from .json_dataset import JSONDataSet \ No newline at end of file + from .json_dataset import JSONDataSet diff --git a/kedro/extras/datasets/pandas/__init__.py b/kedro/extras/datasets/pandas/__init__.py index 9ab6914038..0561d524ea 100644 --- a/kedro/extras/datasets/pandas/__init__.py +++ b/kedro/extras/datasets/pandas/__init__.py @@ -48,8 +48,6 @@ from .csv_dataset import CSVDataSet with suppress(ImportError): from .excel_dataset import ExcelDataSet -with suppress(ImportError): - from .appendable_excel_dataset import AppendableExcelDataSet with suppress(ImportError): from .feather_dataset import FeatherDataSet with suppress(ImportError): diff --git a/kedro/framework/cli/__init__.py b/kedro/framework/cli/__init__.py index b84b0a2464..b46675be9a 100644 --- a/kedro/framework/cli/__init__.py +++ b/kedro/framework/cli/__init__.py @@ -32,4 +32,4 @@ from .cli import main from .utils import command_with_verbosity, load_entry_points -__all__ = ["main", "command_with_verbosity", "load_entry_points"] \ No newline at end of file +__all__ = ["main", "command_with_verbosity", "load_entry_points"] diff --git a/kedro/framework/context/__init__.py b/kedro/framework/context/__init__.py index cb216199cd..bb3aa23972 100644 --- a/kedro/framework/context/__init__.py +++ b/kedro/framework/context/__init__.py @@ -30,6 +30,6 @@ project context. """ -from .context import KedroContext, KedroContextError, load_context +from .context import KedroContext, KedroContextError -__all__ = ["KedroContext", "KedroContextError", "load_context"] \ No newline at end of file +__all__ = ["KedroContext", "KedroContextError"] diff --git a/kedro/io/__init__.py b/kedro/io/__init__.py index c28ac0e597..d5c0ee6bfb 100644 --- a/kedro/io/__init__.py +++ b/kedro/io/__init__.py @@ -42,9 +42,9 @@ ) from .data_catalog import DataCatalog from .data_catalog_with_default import DataCatalogWithDefault -from .lambda_data_set import LambdaDataSet -from .memory_data_set import MemoryDataSet -from .partitioned_data_set import IncrementalDataSet, PartitionedDataSet +from .lambda_dataset import LambdaDataSet +from .memory_dataset import MemoryDataSet +from .partitioned_dataset import IncrementalDataSet, PartitionedDataSet from .transformers import AbstractTransformer __all__ = [ @@ -62,4 +62,4 @@ "MemoryDataSet", "PartitionedDataSet", "Version", -] \ No newline at end of file +] diff --git a/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py b/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py index b756cb8b20..865d2e8e80 100644 --- a/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py +++ b/kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py @@ -34,4 +34,4 @@ __all__ = ["create_pipeline"] -__version__ = "0.1" \ No newline at end of file +__version__ = "0.1"