Merge pull request #153 from Oxid15/develop

0.10.0 - Update

Author: Oxid15

cascade/__init__.py

@@ -15,7 +15,7 @@
 """
 
 
-__version__ = '0.9.0'
+__version__ = '0.10.0'
 __author__ = 'Ilia Moiseev'
 __author_email__ = '[email protected]'
 

cascade/base/__init__.py

@@ -14,10 +14,30 @@
 limitations under the License.
 """
 
-from typing import List, Dict, Any
+from typing import Union, List, Dict, Any
+
+"""
+Single Meta of basic object is just a dict, however Cascade supports
+pipelines with lists of meta.
+
+Do not use Meta when returning from `get_meta` methods! Use PipeMeta instead.
+Meta type alias is designed for better readability and to explicitly state when the
+variable is meta.
+"""
+Meta = Dict[Any, Any]
+
+"""
+This type is used when `get_meta` is called on any Traceable
+"""
+PipeMeta = List[Meta]
+
+"""
+This type described what we can get when reading previously written to meta object
+"""
+MetaFromFile = Union[List[Any], Dict[Any, Any]]
 
-Meta = List[Dict[Any, Any]]
 
 from .meta_handler import MetaHandler, supported_meta_formats
-from .traceable import Traceable, Meta
+from .traceable import Traceable
 from .meta_handler import CustomEncoder as JSONEncoder
+from .history_logger import HistoryLogger

cascade/base/history_logger.py

@@ -0,0 +1,71 @@
+"""
+Copyright 2022 Ilia Moiseev
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+from typing import Any
+from .meta_handler import MetaHandler
+
+
+class HistoryLogger:
+    """
+    An interface to log meta into history files
+    """
+    def __init__(self, filepath: str) -> None:
+        """
+        Parameters
+        ----------
+        filepath: str
+            Path to the history log file
+        """
+        self._mh = MetaHandler()
+        self._log_file = filepath
+
+        if os.path.exists(self._log_file):
+            try:
+                self._log = self._mh.read(self._log_file)
+                if isinstance(self._log, list):
+                    raise RuntimeError(
+                        f'Failed to initialize history logger due to unexpected object'
+                        f' format - log is the instance of list and not dict.'
+                        f' Check your {self._log_file} file')
+                if 'history' not in self._log:
+                    raise RuntimeError(
+                        f'Failed to initialize history logger due to unexpected object'
+                        f' format - "history" key is missing.'
+                        f' Check your {self._log_file} file')
+            except IOError as e:
+                raise IOError(f'Failed to read log file: {self._log_file }') from e
+        else:
+            self._log = {
+                'history': [],
+                'type': 'history'
+            }
+
+    def log(self, obj: Any) -> None:
+        """
+        Logs the state of the object
+
+        Parameters
+        ----------
+        obj: Any
+            Meta data of the object
+        """
+        self._log['history'].append(obj)
+
+        try:
+            self._mh.write(self._log_file, self._log)
+        except IOError as e:
+            raise IOError(f'Failed to write log file: {self._log_file}') from e

cascade/base/meta_handler.py

@@ -17,15 +17,16 @@
 import os
 import json
 import datetime
-from typing import NoReturn, Union, List, Dict, Any
+from typing import NoReturn, Union, Dict, Any
 from json import JSONEncoder
+import deepdiff
 
 import yaml
 import numpy as np
 
-from . import Meta
+from . import MetaFromFile
 
-supported_meta_formats = ('.json', '.yml')
+supported_meta_formats = ('.json', '.yml', '.yaml')
 
 
 class CustomEncoder(JSONEncoder):
@@ -59,14 +60,17 @@ def default(self, obj: Any) -> Any:
         elif isinstance(obj, np.void):
             return None
 
+        elif isinstance(obj, deepdiff.model.PrettyOrderedSet):
+            return list(obj)
+
         return super(CustomEncoder, self).default(obj)
 
-    def obj_to_dict(self, obj: Any) -> Dict:
+    def obj_to_dict(self, obj: Any) -> Any:
         return json.loads(self.encode(obj))
 
 
 class BaseHandler:
-    def read(self, path: str) -> Union[List[Any], Dict[Any, Any]]:
+    def read(self, path: str) -> MetaFromFile:
         raise NotImplementedError()
 
     def write(self, path: str, obj: Any, overwrite: bool = True) -> None:
@@ -83,7 +87,7 @@ def _raise_io_error(self, path: str, exc: Union[Exception, None] = None) -> NoRe
 
 
 class JSONHandler(BaseHandler):
-    def read(self, path: str) -> Union[List[Any], Dict[Any, Any]]:
+    def read(self, path: str) -> MetaFromFile:
         _, ext = os.path.splitext(path)
         if ext == '':
             path += '.json'
@@ -97,7 +101,7 @@ def read(self, path: str) -> Union[List[Any], Dict[Any, Any]]:
                 self._raise_io_error(path, e)
             return meta
 
-    def write(self, path: str, obj: List[Dict], overwrite: bool = True) -> None:
+    def write(self, path: str, obj: Any, overwrite: bool = True) -> None:
         if not overwrite and os.path.exists(path):
             return
 
@@ -106,7 +110,7 @@ def write(self, path: str, obj: List[Dict], overwrite: bool = True) -> None:
 
 
 class YAMLHandler(BaseHandler):
-    def read(self, path: str) -> Union[List[Any], Dict[Any, Any]]:
+    def read(self, path: str) -> MetaFromFile:
         _, ext = os.path.splitext(path)
         if ext == '':
             path += '.yml'
@@ -156,7 +160,7 @@ class MetaHandler:
     """
     Encapsulates the logic of reading and writing metadata to disk.
 
-    Supported read-write formats are `json` and `yml`. Other formats
+    Supported read-write formats are `.json` and `.yml` or `.yaml`. Other formats
     are supported as read-only. For example one can read meta from txt or md file.
 
     Examples
@@ -168,7 +172,7 @@ class MetaHandler:
     >>> mh.write('meta.yml', {'hello': 'world'})
     >>> obj = mh.read('meta.yml')
     """
-    def read(self, path: str) -> Union[List[Any], Dict[Any, Any]]:
+    def read(self, path: str) -> MetaFromFile:
         """
         Reads object from path.
 
@@ -179,7 +183,7 @@ def read(self, path: str) -> Union[List[Any], Dict[Any, Any]]:
 
         Returns
         -------
-            obj: Union[List[Any], Dict[Any, Any]]
+            obj: MetaFromFile
 
         Raises
         ------
@@ -215,7 +219,7 @@ def _get_handler(self, path: str) -> BaseHandler:
         ext = os.path.splitext(path)[-1]
         if ext == '.json':
             return JSONHandler()
-        elif ext == '.yml':
+        elif ext in ('.yml', '.yaml'):
             return YAMLHandler()
         else:
             return TextHandler()

cascade/base/traceable.py

@@ -16,9 +16,8 @@
 
 
 import warnings
-from typing import List, Dict, Union, Any
-
-from . import Meta
+from typing import Dict, Union, Any
+from . import PipeMeta, MetaFromFile
 
 
 class Traceable:
@@ -29,13 +28,13 @@ class Traceable:
     def __init__(
         self,
         *args: Any,
-        meta_prefix: Union[Meta, str, None] = None,
+        meta_prefix: Union[Dict[Any, Any], str, None] = None,
         **kwargs: Any
     ) -> None:
         """
         Parameters
         ----------
-        meta_prefix: Union[Dict, str], optional
+        meta_prefix: Union[Dict[Any, Any], str], optional
             The dictionary that is used to update object's meta in `get_meta` call.
             Due to the call of update can overwrite default values.
             If str - prefix assumed to be path and loaded using MetaHandler.
@@ -51,19 +50,22 @@ def __init__(
         self._meta_prefix = meta_prefix
 
     @staticmethod
-    def _read_meta_from_file(path: str) -> Union[List[Any], Dict[Any, Any]]:
+    def _read_meta_from_file(path: str) -> MetaFromFile:
         from . import MetaHandler
         return MetaHandler().read(path)
 
-    def get_meta(self) -> List[Dict]:
+    def get_meta(self) -> PipeMeta:
         """
         Returns
         -------
-        meta: List[Dict]
-            A list where last element is this object's metadata.
+        meta: PipeMeta
+            A list where first element is this object's metadata.
+            All other elements represent the other stages of pipeline if present.
+
             Meta can be anything that is worth to document about
             the object and its properties.
-            Meta is list to allow the formation of pipelines.
+
+            Meta is a list (see PipeMeta type alias) to allow the formation of pipelines.
         """
         meta = {
             'name': repr(self)
@@ -74,7 +76,7 @@ def get_meta(self) -> List[Dict]:
             self._warn_no_prefix()
         return [meta]
 
-    def update_meta(self, obj: Union[Dict, str]) -> None:
+    def update_meta(self, obj: Union[Dict[Any, Any], str]) -> None:
         """
         Updates `_meta_prefix`, which then updates
         dataset's meta when `get_meta()` is called

cascade/data/apply_modifier.py

@@ -14,7 +14,7 @@
 limitations under the License.
 """
 
-from typing import Callable, Any, List, Dict
+from typing import Callable, Any
 from . import Dataset, Modifier, T
 
 
@@ -23,7 +23,7 @@ class ApplyModifier(Modifier):
     Modifier that maps a function to given dataset's items in a lazy way.
     """
     def __init__(self, dataset: Dataset[T], func: Callable[[T], Any],
-                 *args: List[Any], **kwargs: Dict[Any, Any]) -> None:
+                 *args: Any, **kwargs: Any) -> None:
         """
         Parameters
         ----------

cascade/data/composer.py

@@ -15,10 +15,10 @@
 """
 
 
-from typing import List, Tuple, Dict, Any
+from typing import List, Tuple, Any
 
 from . import SizedDataset
-from ..base import Meta
+from ..base import PipeMeta
 
 
 class Composer(SizedDataset):
@@ -62,7 +62,7 @@ def __getitem__(self, index: int) -> Tuple[Any]:
     def __len__(self) -> int:
         return self._len
 
-    def get_meta(self) -> Meta:
+    def get_meta(self) -> PipeMeta:
         """
         Composer calls `get_meta()` of all its datasets
         """

cascade/data/concatenator.py

@@ -18,7 +18,7 @@
 
 import numpy as np
 from .dataset import SizedDataset, T
-from ..base import Meta
+from ..base import PipeMeta
 
 
 class Concatenator(SizedDataset):
@@ -71,7 +71,7 @@ def __repr__(self) -> str:
         rp = super().__repr__()
         return f'{rp} of\n' + '\n'.join(repr(ds) for ds in self._datasets)
 
-    def get_meta(self) -> Meta:
+    def get_meta(self) -> PipeMeta:
         """
         Concatenator calls `get_meta()` of all its datasets
         """