feat(dev): validation support cont.;

- Moved logic into util method.
- Added new config options for empty resource schemas.
- Renamed some things to make them clearer.

Author: JVickery-TBS

ckanext/xloader/config_declaration.yaml

@@ -119,13 +119,26 @@ groups:
             to True.
         type: bool
         required: false
-      - key: ckanext.xloader.requires_validation
+      - key: ckanext.xloader.validation.requiredSoWaitForReport
         default: False
         example: True
         description: |
             Resources are required to pass validation from the ckanext-validation
             plugin to be able to get xloadered.
         type: bool
         required: false
+      - key: ckanext.xloader.validation.enforceDefaultsIfSchemaMissing
+        default: True
+        example: False
+        description: |
+            Resources are required to have a Schema including the field `validation_status`.
+
+            If this option is set to `False`, Resources that do not have a `validation_status` field will
+            always be submitted to XLoader.
+
+            See https://github.com/frictionlessdata/ckanext-validation?tab=readme-ov-file#changes-in-the-metadata-schema
+            for more details.
+        type: bool
+        required: false
 
 

ckanext/xloader/plugin.py

@@ -82,22 +82,6 @@ def notify(self, entity, operation):
                 or not isinstance(entity, Resource):
             return
 
-        # If the resource requires validation, stop here if validation
-        # has not been performed or did not succeed. The Validation
-        # extension will call resource_patch and this method should
-        # be called again. However, url_changed will not be in the entity
-        # once Validation does the patch.
-        if utils.is_validation_plugin_loaded() and \
-                toolkit.asbool(toolkit.config.get('ckanext.xloader.requires_validation')):
-
-            if entity.__dict__.get('extras', {}).get('validation_status', None) != 'success':
-                log.debug("Skipping xloading resource %s because the "
-                          "resource did not pass validation yet.", entity.id)
-                return
-
-        elif not getattr(entity, 'url_changed', False):
-            return
-
         context = {
             "ignore_auth": True,
         }
@@ -107,15 +91,26 @@ def notify(self, entity, operation):
                 "id": entity.id,
             },
         )
+
+        if utils.awaiting_validation(resource_dict):
+            # If the resource requires validation, stop here if validation
+            # has not been performed or did not succeed. The Validation
+            # extension will call resource_patch and this method should
+            # be called again. However, url_changed will not be in the entity
+            # once Validation does the patch.
+            log.debug("Skipping xloading resource %s because the "
+                      "resource did not pass validation yet.", resource_dict.get('id'))
+            return
+        elif not getattr(entity, 'url_changed', False):
+            # do not submit to xloader if the url has not changed.
+            return
+
         self._submit_to_xloader(resource_dict)
 
     # IResourceController
 
     def after_resource_create(self, context, resource_dict):
-        if utils.is_validation_plugin_loaded() and \
-                toolkit.asbool(toolkit.config.get('ckanext.xloader.requires_validation')) and \
-                resource_dict.get('validation_status', None) != 'success':
-
+        if utils.awaiting_validation(resource_dict):
             log.debug("Skipping xloading resource %s because the "
                       "resource did not pass validation yet.", resource_dict.get('id'))
             return

ckanext/xloader/utils.py

@@ -41,27 +41,63 @@ def is_it_an_xloader_format(cls, format_):
         return format_.lower() in cls._formats
 
 
+def awaiting_validation(res_dict):
+    """
+    Checks the existence of a logic action from the ckanext-validation
+    plugin, thus supporting any extending of the Validation Plugin class.
+
+    Checks ckanext.xloader.validation.requiredSoWaitForReport config
+    option value.
+
+    Checks ckanext.xloader.validation.enforceDefaultsIfSchemaMissing config
+    option value. Then checks the Resource's Schema for the `validation_status` field.
+    """
+    try:
+        p.toolkit.get_action('resource_validation_show')
+        is_validation_plugin_loaded = True
+    except KeyError:
+        is_validation_plugin_loaded = False
+
+    if not is_validation_plugin_loaded or \
+            not p.toolkit.asbool(p.toolkit.config.get('ckanext.xloader.validation.requiredSoWaitForReport', False)):
+        # the validation plugin is not loaded or the validation.requiredSoWaitForReport is turned off
+        return False
+
+    if p.toolkit.asbool(p.toolkit.config.get('ckanext.xloader.validation.enforceDefaultsIfSchemaMissing', True)):
+        # validation.enforceDefaultsIfSchemaMissing is turned on, so we will expect Resources to have `validation_status`
+        if res_dict.get('validation_status', None) != 'success':
+            return True
+        else:
+            return False
+    else:
+        # TODO: check the Resource's schema to see if there is no `validation_status`,
+        # if there is not, then return False, otherwise check if it is `success` or not.
+        return True
+
+    return True
+
+
 def resource_data(id, resource_id, rows=None):
 
     if p.toolkit.request.method == "POST":
-        if is_validation_plugin_loaded() and \
-                p.toolkit.asbool(p.toolkit.config.get('ckanext.xloader.requires_validation')):
-
-            context = {
-                "ignore_auth": True,
-            }
-            resource_dict = p.toolkit.get_action("resource_show")(
-                context,
-                {
-                    "id": resource_id,
-                },
+
+        context = {
+            "ignore_auth": True,
+        }
+        resource_dict = p.toolkit.get_action("resource_show")(
+            context,
+            {
+                "id": resource_id,
+            },
+        )
+
+        if awaiting_validation(resource_dict):
+            h.flash_error(_("Cannot upload resource %s to the DataStore "
+                            "because the resource did not pass validation yet.") % resource_id)
+            return p.toolkit.redirect_to(
+                "xloader.resource_data", id=id, resource_id=resource_id
             )
-            if resource_dict.get('validation_status', None) != 'success':
-                h.flash_error(_("Cannot upload resource %s to the DataStore "
-                                "because the resource did not pass validation yet.") % resource_id)
-                return p.toolkit.redirect_to(
-                    "xloader.resource_data", id=id, resource_id=resource_id
-                )
+
         try:
             p.toolkit.get_action("xloader_submit")(
                 None,
@@ -274,15 +310,3 @@ def datastore_resource_exists(resource_id):
     except p.toolkit.ObjectNotFound:
         return False
     return response or {'fields': []}
-
-
-def is_validation_plugin_loaded():
-    """
-    Checks the existence of a logic action from the ckanext-validation
-    plugin, thus supporting any extending of the Validation Plugin class.
-    """
-    try:
-        p.toolkit.get_action('resource_validation_show')
-        return True
-    except KeyError:
-        return False