Merge pull request #186 from sentinel-hub/develop

Version 0.7.1

Author: iovsn

CHANGELOG.md

@@ -0,0 +1,9 @@
+## [Unreleased]
+
+## [Version 0.7.1] - 2019-02-05
+### Fixed
+- `eolearn.io.SentinelHubInputTask`: evalscript version was not passed to the sentinel-hub service.
+- `eolearn.core.EOWorkflow`: fixed generating task dependencies.
+### Added
+- Processing API docs generation.
+- Introduced CHANGELOG.md.

core/eolearn/core/__init__.py

@@ -18,4 +18,4 @@
 from .utilities import deep_eq, negate_mask, constant_pad, get_common_timestamps, bgr_to_rgb, FeatureParser
 
 
-__version__ = '0.7.0'
+__version__ = '0.7.1'

core/eolearn/core/eoexecution.py

@@ -171,13 +171,13 @@ def _try_add_logging(cls, log_path):
     def _try_remove_logging(cls, log_path, logger, handler, stats):
         if log_path:
             try:
-                logger.info(msg='Pipeline failed.' if cls.STATS_ERROR in stats else 'Pipeline finished.')
+                message = 'EOWorkflow execution {}'.format('failed' if cls.STATS_ERROR in stats else 'finished')
+                logger.debug(message)
                 handler.close()
                 logger.removeHandler(handler)
             except BaseException:
                 pass
 
-
     @classmethod
     def _execute_workflow(cls, process_args):
         """ Handles a single execution of a workflow

core/eolearn/core/eoworkflow.py

@@ -268,14 +268,23 @@ def get_tasks(self):
         """ Returns an ordered dictionary {task_name: task} of all tasks within this workflow
 
         :return: Ordered dictionary with key being task_name (str) and an instance of a corresponding task from this
-        workflow
+            workflow. The order of tasks is the same as in which they will be executed.
         :rtype: OrderedDict
         """
-        tasks = collections.OrderedDict()
+        task_dict = collections.OrderedDict()
         for dep in self.ordered_dependencies:
-            tasks[dep.name] = dep.task
+            task_name = dep.name
 
-        return tasks
+            if task_name in task_dict:
+                count = 0
+                while dep.get_custom_name(count) in task_dict:
+                    count += 1
+
+                task_name = dep.get_custom_name(count)
+
+            task_dict[task_name] = dep.task
+
+        return task_dict
 
     def get_dot(self):
         """ Generates the DOT description of the underlying computational graph
@@ -327,7 +336,7 @@ def __init__(self, *tasks, **kwargs):
         tasks = [self._parse_task(task) for task in tasks]
         tasks = self._make_tasks_unique(tasks)
 
-        dependencies = [(task, [tasks[idx - 1][0]] if idx > 0 else []) for idx, (task, name) in enumerate(tasks)]
+        dependencies = [(task, [tasks[idx - 1][0]] if idx > 0 else [], name) for idx, (task, name) in enumerate(tasks)]
         super().__init__(dependencies, **kwargs)
 
     @staticmethod
@@ -396,6 +405,13 @@ def set_name(self, name):
         """
         self.name = name
 
+    def get_custom_name(self, number=0):
+        """ Provides custom task name according to given number. E.g. FooTask -> FooTask
+        """
+        if number:
+            return '{}_{}'.format(self.name, number)
+        return self.name
+
 
 class WorkflowResults(collections.Mapping):
     """ The result of a workflow is an (immutable) dictionary mapping [1] from EOTasks to results of the workflow.

core/eolearn/tests/test_eoworkflow.py

@@ -108,48 +108,48 @@ def test_workflow_arguments(self):
 
     def test_linear_workflow(self):
         in_task = InputTask()
+        in_task_name = 'My input task'
         inc_task = Inc()
         pow_task = Pow()
-        eow = LinearWorkflow((in_task, 'task name'), inc_task, inc_task, pow_task)
+        eow = LinearWorkflow((in_task, in_task_name), inc_task, inc_task, pow_task)
         res = eow.execute({
             in_task: {'val': 2},
             inc_task: {'d': 2},  # Note that this will assign value only to one instance of Inc task
             pow_task: {'n': 3}
         })
         self.assertEqual(res[pow_task], (2 + 2 + 1) ** 3)
 
+        task_map = eow.get_tasks()
+        self.assertTrue(in_task_name in task_map, "A task with name '{}' should be amongst tasks".format(in_task_name))
+        self.assertEqual(task_map[in_task_name], in_task,
+                         "A task with name '{}' should map into {}".format(in_task_name, in_task))
+
     def test_get_tasks(self):
         in_task = InputTask()
         inc_task = Inc()
-        pow_task = Pow()
 
-        task_names = ['InputTask', 'Inc', 'Pow']
-        workflow_tasks = [in_task, inc_task, pow_task]
-        eow = LinearWorkflow(*workflow_tasks)
+        task_names = ['InputTask', 'Inc', 'Inc_1', 'Inc_2']
+        eow = LinearWorkflow(in_task, inc_task, inc_task, inc_task)
 
         returned_tasks = eow.get_tasks()
 
         # check if tasks are present
-        for task_name in task_names:
-            self.assertIn(task_name, returned_tasks.keys())
+        self.assertEqual(sorted(task_names), sorted(returned_tasks))
 
         # check if tasks still work
         arguments_dict = {
             in_task: {'val': 2},
-            inc_task: {'d': 2},
-            pow_task: {'n': 3}
+            inc_task: {'d': 2}
         }
 
         res_workflow = eow.execute(arguments_dict)
-        res_workflow_value = [res_workflow[key] for key in res_workflow.keys()][0]
+        res_workflow_value = list(res_workflow.values())
 
-        for idx, task in enumerate(workflow_tasks):
-            if idx == 0:
-                res_tasks_value = task.execute(**arguments_dict[task])
-            else:
-                res_tasks_value = task.execute(res_tasks_value, **arguments_dict[task])
+        res_tasks_values = []
+        for idx, task in enumerate(returned_tasks.values()):
+            res_tasks_values = [task.execute(*res_tasks_values, **arguments_dict.get(task, {}))]
 
-        self.assertEqual(res_workflow_value, res_tasks_value)
+        self.assertEqual(res_workflow_value, res_tasks_values)
 
     def test_trivial_workflow(self):
         task = DummyTask()

docs/source/eolearn.io.processing_api.rst

@@ -0,0 +1,7 @@
+eolearn.io.processing\_api
+===============================
+
+.. automodule:: eolearn.io.processing_api
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/eolearn.io.rst

@@ -13,3 +13,4 @@ Submodules:
    eolearn.io.geopedia
    eolearn.io.local_io
    eolearn.io.sentinelhub_service
+   eolearn.io.processing_api

io/eolearn/io/__init__.py

@@ -9,4 +9,4 @@
 from .local_io import ExportToTiff, ImportFromTiff
 from .processing_api import SentinelHubInputTask, SentinelHubDemTask
 
-__version__ = '0.7.0'
+__version__ = '0.7.1'

io/eolearn/io/processing_api.py

@@ -206,6 +206,8 @@ def generate_evalscript(self):
         """ Generate the evalscript to be passed with the request, based on chosen bands
         """
         evalscript = """
+            //VERSION=3
+
             function setup() {{
                 return {{
                     input: [{{
@@ -363,6 +365,8 @@ def _build_payloads(self, bbox, size_x, size_y, timestamp, time_interval):
         """ Build payloads for the requests to the service
         """
         evalscript = """
+            //VERSION=3
+
             function setup() {
                 return {
                     input: ["DEM"],

setup.py

@@ -22,7 +22,7 @@ def parse_requirements(file):
 setup(
     name='eo-learn',
     python_requires='>=3.5',
-    version='0.7.0',
+    version='0.7.1',
     description='Earth observation processing framework for machine learning in Python',
     long_description=get_long_description(),
     long_description_content_type='text/markdown',
@@ -33,14 +33,14 @@ def parse_requirements(file):
     packages=[],
     include_package_data=True,
     install_requires=[
-        'eo-learn-core>=0.7.0',
+        'eo-learn-core>=0.7.1',
         'eo-learn-coregistration>=0.7.0',
         'eo-learn-features>=0.7.0',
         'eo-learn-geometry>=0.7.0',
-        'eo-learn-io>=0.7.0',
+        'eo-learn-io>=0.7.1',
         'eo-learn-mask>=0.7.0',
         'eo-learn-ml-tools>=0.7.0',
-        'eo-learn-visualization>=0.7.0'
+        'eo-learn-visualization>=0.7.1'
     ],
     extras_require={
         'DEV': parse_requirements('requirements-dev.txt')

visualization/eolearn/visualization/__init__.py

@@ -10,4 +10,4 @@
 except ImportError:
     pass
 
-__version__ = '0.7.0'
+__version__ = '0.7.1'

visualization/eolearn/visualization/eoworkflow_visualization.py

@@ -73,11 +73,7 @@ def _get_dep_to_dot_name_mapping(dependencies):
 
         dep_to_dot_name = {}
         for dot_name, deps in dot_name_to_deps.items():
-            if len(deps) == 1:
-                dep_to_dot_name[deps[0]] = dot_name
-                continue
-
             for idx, dep in enumerate(deps):
-                dep_to_dot_name[dep] = dot_name + str(idx)
+                dep_to_dot_name[dep] = dep.get_custom_name(idx)
 
         return dep_to_dot_name