feat(dev): better logic and tests;

- Added automated tests.
- Changed the logic to be better and more clear.

Author: JVickery-TBS

ckanext/xloader/plugin.py

@@ -82,22 +82,6 @@ def notify(self, entity, operation):
                 or not isinstance(entity, Resource):
             return
 
-        # If the resource requires validation, stop here if validation
-        # has not been performed or did not succeed. The Validation
-        # extension will call resource_patch and this method should
-        # be called again. However, url_changed will not be in the entity
-        # once Validation does the patch.
-        if utils.awaiting_validation() and \
-                toolkit.asbool(toolkit.config.get('ckanext.xloader.requires_validation')):
-
-            if entity.__dict__.get('extras', {}).get('validation_status', None) != 'success':
-                log.debug("Skipping xloading resource %s because the "
-                          "resource did not pass validation yet.", entity.id)
-                return
-
-        elif not getattr(entity, 'url_changed', False):
-            return
-
         context = {
             "ignore_auth": True,
         }

ckanext/xloader/tests/test_plugin.py

@@ -58,6 +58,88 @@ def test_submit_when_url_changes(self, monkeypatch):
 
         assert func.called
 
+    @pytest.mark.ckan_config("ckanext.xloader.validation.requires_successful_report", True)
+    def test_require_validation(self, monkeypatch):
+        func = mock.Mock()
+        monkeypatch.setitem(_actions, "xloader_submit", func)
+
+        mock_resource_validation_show = mock.Mock()
+        monkeypatch.setitem(_actions, "resource_validation_show", mock_resource_validation_show)
+
+        dataset = factories.Dataset()
+
+        resource = helpers.call_action(
+            "resource_create",
+            {},
+            package_id=dataset["id"],
+            url="http://example.com/file.csv",
+            format="CSV",
+            validation_status='failure',
+        )
+
+        assert not func.called  # because of the validation_status not being `success`
+        func.called = None # reset
+
+        helpers.call_action(
+            "resource_update",
+            {},
+            id=resource["id"],
+            package_id=dataset["id"],
+            url="http://example.com/file2.csv",
+            format="CSV",
+            validation_status='success',
+        )
+
+        assert func.called  # because of the validation_status is `success`
+
+    @pytest.mark.ckan_config("ckanext.xloader.validation.requires_successful_report", True)
+    @pytest.mark.ckan_config("ckanext.xloader.validation.enforce_schema", False)
+    def test_enforce_validation_schema(self, monkeypatch):
+        func = mock.Mock()
+        monkeypatch.setitem(_actions, "xloader_submit", func)
+
+        mock_resource_validation_show = mock.Mock()
+        monkeypatch.setitem(_actions, "resource_validation_show", mock_resource_validation_show)
+
+        dataset = factories.Dataset()
+
+        resource = helpers.call_action(
+            "resource_create",
+            {},
+            package_id=dataset["id"],
+            url="http://example.com/file.csv",
+            schema='',
+            validation_status='',
+        )
+
+        assert func.called  # because of the schema being empty
+        func.called = None # reset
+
+        helpers.call_action(
+            "resource_update",
+            {},
+            id=resource["id"],
+            package_id=dataset["id"],
+            url="http://example.com/file2.csv",
+            schema='https://example.com/schema.json',
+            validation_status='failure',
+        )
+
+        assert not func.called  # because of the validation_status not being `success` and there is a schema
+        func.called = None # reset
+
+        helpers.call_action(
+            "resource_update",
+            {},
+            package_id=dataset["id"],
+            id=resource["id"],
+            url="http://example.com/file3.csv",
+            schema='https://example.com/schema.json',
+            validation_status='success',
+        )
+
+        assert func.called  # because of the validation_status is `success` and there is a schema
+
     def _pending_task(self, resource_id):
         return {
             "entity_id": resource_id,

ckanext/xloader/utils.py

@@ -76,14 +76,17 @@ def awaiting_validation(res_dict):
         log.warning('ckanext.xloader.validation.requires_successful_report requires the ckanext-validation plugin to be activated.')
         return False
 
-    if p.toolkit.asbool(config.get('ckanext.xloader.validation.enforce_schema', True)):
-        # validation.enforce_schema is turned on, so we will always look for `validation_status`
-        if res_dict.get('validation_status', None) != 'success':
-            return True
-
-    # validation.enforce_schema is turned off, so if the Resource
-    # does not have a Validation Schema, we will treat it like
-    # it does not require Validation.
+    if p.toolkit.asbool(config.get('ckanext.xloader.validation.enforce_schema', True)) \
+            and res_dict.get('validation_status', None) != 'success':
+        # validation.enforce_schema is turned on, and there is no successful report.
+        return True
+
+    elif res_dict.get('schema', None) and res_dict.get('validation_status', None) != 'success':
+        # validation.enforce_schema is turned off, and there is a Validation Schema and no successful report.
+        return True
+
+    # at this point, we can assume that the Resource is not waiting for Validation.
+    # or that the Resource does not have a Validation Schema and we are not enforcing schemas.
     return False