Allow JSON uploads re #10798

archesproject · May 6, 2024 · 35b4f1b · 35b4f1b
1 parent e33a342
commit 35b4f1b
Show file tree

Hide file tree

Showing 6 changed files with 34 additions and 25 deletions.
diff --git a/arches/app/etl_modules/base_import_module.py b/arches/app/etl_modules/base_import_module.py
@@ -242,10 +242,12 @@ def read(self, request):
         validator = FileValidator()
         if len(validator.validate_file_type(content)) > 0:
             return {
-                "status": 400,
                 "success": False,
-                "title": _("Invalid excel file/zip specified"),
-                "message": _("Upload a valid excel file"),
+                "data": FileValidationError(
+                    title=_("Invalid excel file/zip specified"),
+                    message=_("Upload a valid excel file"),
+                    code=400,
+                )
             }
         if content.name.split(".")[-1].lower() == "zip":
             with zipfile.ZipFile(content, "r") as zip_ref:

diff --git a/arches/app/etl_modules/jsonld_importer.py b/arches/app/etl_modules/jsonld_importer.py
@@ -42,12 +42,13 @@ def read(self, request):
             }
         }
         validator = FileValidator()
-        if validator.validate_file_type(content):
+        if (file_type_errors := validator.validate_file_type(content)):
             return {
-                "status": 400,
                 "success": False,
-                "title": _("Invalid Uploaded File"),
-                "message": _("Upload a valid zip file"),
+                "data": FileValidationError(
+                    message=", ".join(file_type_errors),
+                    code=400,
+                )
             }
 
         with zipfile.ZipFile(content, "r") as zip_ref:

diff --git a/arches/app/utils/file_validator.py b/arches/app/utils/file_validator.py
@@ -1,4 +1,5 @@
 import io
+import json
 import logging
 import zipfile
 from django.conf import settings
@@ -14,15 +15,17 @@ def __init__(self):
 
     def test_unknown_filetypes(self, file, extension=None):
         errors = []
-        if extension in settings.FILE_TYPES:
-            if extension == "xlsx":
+        match extension:
+            case "DS_Store":
+                self.logger.log(logging.WARN, "DS_Store file encountered, proceeding with caution.")
+            case _ if extension not in settings.FILE_TYPES:
+                errors.append(f"File type is not permitted: {extension}")
+            case "xlsx":
                 try:
                     load_workbook(io.BytesIO(file))
                 except (InvalidFileException, zipfile.BadZipFile):
-                    error = "Invalid xlsx workbook"
-                    self.logger.log(logging.ERROR, error)
-                    errors.append(error)
-            elif extension == "csv":
+                    errors.append("Invalid xlsx workbook")
+            case "csv":
                 try:
                     datareader = csv.reader(file.decode("utf-8").splitlines(), delimiter=",")
                     length = None
@@ -32,17 +35,17 @@ def test_unknown_filetypes(self, file, extension=None):
                         elif length is None:
                             length = len(row)
                 except csv.Error:
-                    error = "Invalid csv file"
-                    self.logger.log(logging.ERROR, error)
-                    errors.append(error)
-            else:
-                error = "Cannot validate file"
-                self.logger.log(logging.ERROR, error)
-                errors.append(error)
-        else:
-            error = "File type is not permitted"
+                    errors.append("Invalid csv file")
+            case "json":
+                try:
+                    json.load(io.BytesIO(file))
+                except json.decoder.JSONDecodeError:
+                    errors.append("Invalid json file")
+            case _:
+                errors.append("Cannot validate file")
+
+        for error in errors:
             self.logger.log(logging.ERROR, error)
-            errors.append(error)
 
         return errors
 

diff --git a/arches/install/arches-templates/project_name/settings.py-tpl b/arches/install/arches-templates/project_name/settings.py-tpl
@@ -37,7 +37,7 @@ SEARCH_COMPONENT_LOCATIONS.append('{{ project_name }}.search_components')
 LOCALE_PATHS.append(os.path.join(APP_ROOT, 'locale'))
 
 FILE_TYPE_CHECKING = False
-FILE_TYPES = ["bmp", "gif", "jpg", "jpeg", "pdf", "png", "psd", "rtf", "tif", "tiff", "xlsx", "csv", "zip"]
+FILE_TYPES = ["bmp", "gif", "jpg", "jpeg", "json", "pdf", "png", "psd", "rtf", "tif", "tiff", "xlsx", "csv", "zip"]
 FILENAME_GENERATOR = "arches.app.utils.storage_filename_generator.generate_filename"
 UPLOADED_FILES_DIR = "uploadedfiles"
 

diff --git a/arches/settings.py b/arches/settings.py
@@ -610,7 +610,7 @@
 ]
 
 FILE_TYPE_CHECKING = False
-FILE_TYPES = ["bmp", "gif", "jpg", "jpeg", "pdf", "png", "psd", "rtf", "tif", "tiff", "xlsx", "csv", "zip"]
+FILE_TYPES = ["bmp", "gif", "jpg", "jpeg", "json", "pdf", "png", "psd", "rtf", "tif", "tiff", "xlsx", "csv", "zip"]
 FILENAME_GENERATOR = "arches.app.utils.storage_filename_generator.generate_filename"
 UPLOADED_FILES_DIR = "uploadedfiles"
 

diff --git a/releases/7.6.0.md b/releases/7.6.0.md
@@ -9,6 +9,8 @@ Arches 7.6.0 Release Notes
 
 - Plugins now support the configuration boolean `is_standalone`. Standalone plugins do not appear in the sidebar, and do not display the sidebar or application header.
 
+- 10862 JSON-LD bulk import module
+
 ### Performance Improvements
 - 10453 Reduce queries for related objects when indexing resources
 
@@ -202,6 +204,7 @@ Minor incompatibilities:
             },
         }
         ```
+    3. Add `"json"` to `FILE_TYPES` if you wish to use the JSON-LD bulk import module.
 
 9. Run `python manage.py updateproject`