Merge pull request #1238 from quantumblacklabs/merge-master-to-develop

[AUTO-MERGE] Merge master into develop via merge-master-to-develop

.gitignore

@@ -114,6 +114,7 @@ celerybeat-schedule
 
 # Environments
 .env
+.envrc
 .venv
 env/
 venv/

RELEASE.md

@@ -118,6 +118,7 @@
 * Bumped minimum required `fsspec` version to 2021.04.
 * Fixed the `kedro install` and `kedro build-reqs` flows when uninstalled dependencies are present in a project's `settings.py`, `context.py` or `hooks.py` ([Issue #829](https://github.com/quantumblacklabs/kedro/issues/829)).
 * Imports are now refactored at `kedro pipeline package` and `kedro pipeline pull` time, so that _aliasing_ a modular pipeline doesn't break it.
+* Pinned `dynaconf` to `<3.1.6` because the method signature for `_validate_items` changed which is used in Kedro.
 
 ## Minor breaking changes to the API
 

docs/source/05_data/01_data_catalog.md

@@ -361,7 +361,7 @@ CSVDataSet(
 
 ## Loading multiple datasets that have similar configuration
 
-You may encounter situations where your datasets use the same file format, load and save arguments, and are stored in the same folder. YAML has a [built-in syntax](https://yaml.org/spec/1.2/#Syntax) for factorising parts of a YAML file, which means that you can decide what is generalisable across your datasets so that you do not have to spend time copying and pasting dataset configurations in `catalog.yml`.
+You may encounter situations where your datasets use the same file format, load and save arguments, and are stored in the same folder. YAML has a [built-in syntax](https://yaml.org/spec/1.2.1/#Syntax) for factorising parts of a YAML file, which means that you can decide what is generalisable across your datasets so that you do not have to spend time copying and pasting dataset configurations in `catalog.yml`.
 
 You can see this in the following example:
 

kedro/config/__init__.py

@@ -30,9 +30,18 @@
 configuration from different file formats.
 """
 
+from .abstract_config import (
+    AbstractConfigLoader,
+    BadConfigException,
+    MissingConfigException,
+)
+from .config import ConfigLoader
+from .templated_config import TemplatedConfigLoader
 
-from .abstract_config import AbstractConfigLoader  # NOQA
-from .abstract_config import BadConfigException  # NOQA
-from .abstract_config import MissingConfigException  # NOQA
-from .config import ConfigLoader  # NOQA
-from .templated_config import TemplatedConfigLoader  # NOQA
+__all__ = [
+    "AbstractConfigLoader",
+    "BadConfigException",
+    "ConfigLoader",
+    "MissingConfigException",
+    "TemplatedConfigLoader",
+]

kedro/extras/datasets/api/__init__.py

@@ -36,4 +36,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .api_dataset import APIDataSet  # NOQA
+    from .api_dataset import APIDataSet

kedro/extras/datasets/biosequence/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .biosequence_dataset import BioSequenceDataSet  # NOQA
+    from .biosequence_dataset import BioSequenceDataSet

kedro/extras/datasets/dask/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .parquet_dataset import ParquetDataSet  # NOQA
+    from .parquet_dataset import ParquetDataSet

kedro/extras/datasets/email/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .message_dataset import EmailMessageDataSet  # NOQA
+    from .message_dataset import EmailMessageDataSet

kedro/extras/datasets/geopandas/__init__.py

@@ -32,4 +32,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .geojson_dataset import GeoJSONDataSet  # NOQA
+    from .geojson_dataset import GeoJSONDataSet

kedro/extras/datasets/holoviews/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .holoviews_writer import HoloviewsWriter  # NOQA
+    from .holoviews_writer import HoloviewsWriter

kedro/extras/datasets/json/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .json_dataset import JSONDataSet  # NOQA
+    from .json_dataset import JSONDataSet

kedro/extras/datasets/matplotlib/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .matplotlib_writer import MatplotlibWriter  # NOQA
+    from .matplotlib_writer import MatplotlibWriter

kedro/extras/datasets/pandas/__init__.py

@@ -45,20 +45,20 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .csv_dataset import CSVDataSet  # NOQA
+    from .csv_dataset import CSVDataSet
 with suppress(ImportError):
-    from .excel_dataset import ExcelDataSet  # NOQA
+    from .excel_dataset import ExcelDataSet
 with suppress(ImportError):
-    from .feather_dataset import FeatherDataSet  # NOQA
+    from .feather_dataset import FeatherDataSet
 with suppress(ImportError):
-    from .gbq_dataset import GBQTableDataSet  # NOQA
+    from .gbq_dataset import GBQTableDataSet
 with suppress(ImportError):
-    from .hdf_dataset import HDFDataSet  # NOQA
+    from .hdf_dataset import HDFDataSet
 with suppress(ImportError):
-    from .json_dataset import JSONDataSet  # NOQA
+    from .json_dataset import JSONDataSet
 with suppress(ImportError):
-    from .parquet_dataset import ParquetDataSet  # NOQA
+    from .parquet_dataset import ParquetDataSet
 with suppress(ImportError):
-    from .sql_dataset import SQLQueryDataSet, SQLTableDataSet  # NOQA
+    from .sql_dataset import SQLQueryDataSet, SQLTableDataSet
 with suppress(ImportError):
-    from .xml_dataset import XMLDataSet  # NOQA
+    from .xml_dataset import XMLDataSet

kedro/extras/datasets/pickle/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .pickle_dataset import PickleDataSet  # NOQA
+    from .pickle_dataset import PickleDataSet

kedro/extras/datasets/pillow/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .image_dataset import ImageDataSet  # NOQA
+    from .image_dataset import ImageDataSet

kedro/extras/datasets/plotly/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .plotly_dataset import PlotlyDataSet  # NOQA
+    from .plotly_dataset import PlotlyDataSet

kedro/extras/datasets/spark/__init__.py

@@ -33,8 +33,8 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .spark_dataset import SparkDataSet  # NOQA
+    from .spark_dataset import SparkDataSet
 with suppress(ImportError):
-    from .spark_hive_dataset import SparkHiveDataSet  # NOQA
+    from .spark_hive_dataset import SparkHiveDataSet
 with suppress(ImportError):
-    from .spark_jdbc_dataset import SparkJDBCDataSet  # NOQA
+    from .spark_jdbc_dataset import SparkJDBCDataSet

kedro/extras/datasets/tensorflow/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .tensorflow_model_dataset import TensorFlowModelDataset  # NOQA
+    from .tensorflow_model_dataset import TensorFlowModelDataset

kedro/extras/datasets/text/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .text_dataset import TextDataSet  # NOQA
+    from .text_dataset import TextDataSet

kedro/extras/datasets/tracking/__init__.py

@@ -34,6 +34,6 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from kedro.extras.datasets.tracking.metrics_dataset import MetricsDataSet  # NOQA
+    from kedro.extras.datasets.tracking.metrics_dataset import MetricsDataSet
 with suppress(ImportError):
-    from kedro.extras.datasets.tracking.json_dataset import JSONDataSet  # NOQA
+    from kedro.extras.datasets.tracking.json_dataset import JSONDataSet

kedro/extras/datasets/yaml/__init__.py

@@ -33,4 +33,4 @@
 from contextlib import suppress
 
 with suppress(ImportError):
-    from .yaml_dataset import YAMLDataSet  # NOQA
+    from .yaml_dataset import YAMLDataSet

kedro/extras/logging/__init__.py

@@ -30,4 +30,6 @@
 This module contains a logging handler class which produces coloured logs.
 """
 
-from .color_logger import ColorHandler  # NOQA
+from .color_logger import ColorHandler
+
+__all__ = ["ColorHandler"]

kedro/extras/transformers/__init__.py

@@ -28,5 +28,7 @@
 
 """``kedro.extras.transformers`` is the home of Kedro's dataset transformers."""
 
-from .memory_profiler import ProfileMemoryTransformer  # NOQA
-from .time_profiler import ProfileTimeTransformer  # NOQA
+from .memory_profiler import ProfileMemoryTransformer
+from .time_profiler import ProfileTimeTransformer
+
+__all__ = ["ProfileMemoryTransformer", "ProfileTimeTransformer"]

kedro/framework/cli/__init__.py

@@ -29,5 +29,7 @@
 """``kedro.framework.cli`` implements commands available from Kedro's CLI.
 """
 
-from .cli import main  # NOQA
-from .utils import command_with_verbosity, load_entry_points  # NOQA
+from .cli import main
+from .utils import command_with_verbosity, load_entry_points
+
+__all__ = ["main", "command_with_verbosity", "load_entry_points"]

kedro/framework/cli/hooks/__init__.py

@@ -26,5 +26,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """``kedro.framework.cli.hooks`` provides primitives to use hooks to extend KedroCLI's behaviour"""
-from .manager import CLIHooksManager  # NOQA
-from .markers import cli_hook_impl  # NOQA
+from .manager import CLIHooksManager
+from .markers import cli_hook_impl
+
+__all__ = ["CLIHooksManager", "cli_hook_impl"]

kedro/framework/context/__init__.py

@@ -30,5 +30,6 @@
 project context.
 """
 
-from .context import KedroContext  # NOQA
-from .context import KedroContextError  # NOQA
+from .context import KedroContext, KedroContextError
+
+__all__ = ["KedroContext", "KedroContextError"]

kedro/framework/hooks/__init__.py

@@ -26,5 +26,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """``kedro.framework.hooks`` provides primitives to use hooks to extend KedroContext's behaviour"""
-from .manager import get_hook_manager  # NOQA
-from .markers import hook_impl  # NOQA
+from .manager import get_hook_manager
+from .markers import hook_impl
+
+__all__ = ["get_hook_manager", "hook_impl"]

kedro/framework/session/__init__.py

@@ -29,4 +29,6 @@
 """``kedro.framework.session`` provides access to KedroSession responsible for
 project lifecycle.
 """
-from .session import KedroSession, get_current_session  # NOQA
+from .session import KedroSession, get_current_session
+
+__all__ = ["KedroSession", "get_current_session"]

kedro/io/__init__.py

@@ -31,17 +31,35 @@
 which allows implementation of various ``AbstractDataSet``s.
 """
 
-from .cached_dataset import CachedDataSet  # NOQA
-from .core import AbstractDataSet  # NOQA
-from .core import AbstractVersionedDataSet  # NOQA
-from .core import DataSetAlreadyExistsError  # NOQA
-from .core import DataSetError  # NOQA
-from .core import DataSetNotFoundError  # NOQA
-from .core import Version  # NOQA
-from .data_catalog import DataCatalog  # NOQA
-from .data_catalog_with_default import DataCatalogWithDefault  # NOQA
-from .lambda_dataset import LambdaDataSet  # NOQA
-from .memory_dataset import MemoryDataSet  # NOQA
-from .partitioned_dataset import IncrementalDataSet  # NOQA
-from .partitioned_dataset import PartitionedDataSet  # NOQA
-from .transformers import AbstractTransformer  # NOQA
+from .cached_dataset import CachedDataSet
+from .core import (
+    AbstractDataSet,
+    AbstractVersionedDataSet,
+    DataSetAlreadyExistsError,
+    DataSetError,
+    DataSetNotFoundError,
+    Version,
+)
+from .data_catalog import DataCatalog
+from .data_catalog_with_default import DataCatalogWithDefault
+from .lambda_dataset import LambdaDataSet
+from .memory_dataset import MemoryDataSet
+from .partitioned_dataset import IncrementalDataSet, PartitionedDataSet
+from .transformers import AbstractTransformer
+
+__all__ = [
+    "AbstractDataSet",
+    "AbstractTransformer",
+    "AbstractVersionedDataSet",
+    "CachedDataSet",
+    "DataCatalog",
+    "DataCatalogWithDefault",
+    "DataSetAlreadyExistsError",
+    "DataSetError",
+    "DataSetNotFoundError",
+    "IncrementalDataSet",
+    "LambdaDataSet",
+    "MemoryDataSet",
+    "PartitionedDataSet",
+    "Version",
+]

kedro/pipeline/__init__.py

@@ -30,6 +30,8 @@
 data-driven pipelines.
 """
 
-from .modular_pipeline import pipeline  # NOQA
-from .node import node  # NOQA
-from .pipeline import Pipeline  # NOQA
+from .modular_pipeline import pipeline
+from .node import node
+from .pipeline import Pipeline
+
+__all__ = ["pipeline", "node", "Pipeline"]

kedro/runner/__init__.py

@@ -30,7 +30,15 @@
 to execute ``Pipeline`` instances.
 """
 
-from .parallel_runner import ParallelRunner  # NOQA
-from .runner import AbstractRunner, run_node  # NOQA
-from .sequential_runner import SequentialRunner  # NOQA
-from .thread_runner import ThreadRunner  # NOQA
+from .parallel_runner import ParallelRunner
+from .runner import AbstractRunner, run_node
+from .sequential_runner import SequentialRunner
+from .thread_runner import ThreadRunner
+
+__all__ = [
+    "AbstractRunner",
+    "ParallelRunner",
+    "SequentialRunner",
+    "ThreadRunner",
+    "run_node",
+]

kedro/templates/pipeline/{{ cookiecutter.pipeline_name }}/__init__.py

@@ -30,6 +30,8 @@
 generated using Kedro {{ cookiecutter.kedro_version }}
 """
 
-from .pipeline import create_pipeline  # NOQA
+from .pipeline import create_pipeline
+
+__all__ = ["create_pipeline"]
 
 __version__ = "0.1"

kedro/templates/project/{{ cookiecutter.repo_name }}/.gitignore

@@ -144,6 +144,7 @@ celerybeat-schedule
 
 # Environments
 .env
+.envrc
 .venv
 env/
 venv/

kedro/versioning/__init__.py

@@ -30,4 +30,6 @@
 capturing information required to reproduce a Kedro run.
 """
 
-from .journal import Journal  # NOQA
+from .journal import Journal
+
+__all__ = ["Journal"]

requirements.txt

@@ -2,7 +2,7 @@ anyconfig~=0.10.0
 cachetools~=4.1
 click<8.0
 cookiecutter~=1.7.0
-dynaconf~=3.1.2
+dynaconf<3.1.6 # Pinned because Dynaconf broke a method signature in 3.1.6 used in Kedro
 fsspec>=2021.04, <2022.01  # Upper bound set arbitrarily, to be reassessed in early 2022
 gitpython~=3.0
 jmespath>=0.9.5, <1.0