[PULP-817] Adjustments to make imp/exp work with django-imp-exp 4.x.

closes #5324.

Author: ggainey

CHANGES/5324.feature

@@ -0,0 +1 @@
+Adapted PulpImport/Export to allow update django-import-export==4.x.

pulpcore/app/importexport.py

@@ -48,6 +48,10 @@ def _write_export(the_tarfile, resource, dest_dir=None):
     # the data in batches to memory and concatenate the json lists via string manipulation.
     with tempfile.NamedTemporaryFile(dir=".", mode="w", encoding="utf8") as temp_file:
         if isinstance(resource.queryset, QuerySet):
+            # If we don't have any of "these" - skip writing
+            if resource.queryset.count() == 0:
+                return
+
             temp_file.write("[")
 
             def process_batch(batch):

pulpcore/app/modelresource.py

@@ -62,8 +62,8 @@ def before_import_row(self, row, **kwargs):
         # the export converts None to blank strings but sha384 and sha512 have unique constraints
         # that get triggered if they are blank. convert checksums back into None if they are blank.
         for checksum in ALL_KNOWN_CONTENT_CHECKSUMS:
-            if row[checksum] == "":
-                row[checksum] = None
+            if row[checksum] == "" or not checksum in settings.ALLOWED_CONTENT_CHECKSUMS:
+                del row[checksum]
 
     def set_up_queryset(self):
         """
@@ -109,7 +109,7 @@ def get_queryset(self, value, row, *args, **kwargs):
         return qs
 
     def render(self, value, obj=None, **kwargs):
-        return value.sha256
+        return value.sha256 if value else ""
 
 
 class ContentArtifactResource(QueryModelResource):

pulpcore/app/tasks/importer.py

@@ -233,6 +233,9 @@ def _import_file(fpath, resource_class, retry=False):
     """
     try:
         log.info(f"Importing file {fpath}.")
+        if not os.path.isfile(fpath):
+            log.info("...empty - skipping.")
+            return []
         with open(fpath, "r") as json_file:
             resource = resource_class()
             log.info(f"...Importing resource {resource.__class__.__name__}.")
@@ -241,6 +244,8 @@ def _import_file(fpath, resource_class, retry=False):
             # overlapping content.
             for batch_str in _impfile_iterator(json_file):
                 data = Dataset().load(StringIO(batch_str))
+                if not data:
+                    return []
                 if retry:
                     curr_attempt = 1
 
@@ -272,6 +277,7 @@ def _import_file(fpath, resource_class, retry=False):
                         try:
                             a_result = resource.import_data(data, raise_errors=True)
                         except Exception as e:  # noqa log on ANY exception and then re-raise
+                            log.error(e)
                             log.error(f"FATAL import-failure importing {fpath}")
                             raise
                 else:

pulpcore/plugin/importexport.py

@@ -41,6 +41,9 @@ def set_up_queryset(self):
     def dehydrate_pulp_domain(self, content):
         return str(content.pulp_domain_id)
 
+    def render(self, value, obj=None, **kwargs):
+        return super().render(value, obj, coerce_to_string=False, **kwargs)
+
     def __init__(self, repo_version=None):
         self.repo_version = repo_version
         if repo_version:

pulpcore/tests/functional/api/using_plugin/test_filesystemexport.py

@@ -16,8 +16,6 @@
 NUM_REPOS = 1
 NUM_EXPORTERS = 4
 
-pytestmark = pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domains do not support export.")
-
 
 @pytest.fixture
 def create_exporter(pulpcore_bindings, gen_object_with_cleanup, add_to_filesystem_cleanup):

pulpcore/tests/functional/api/using_plugin/test_pulpimport.py

@@ -21,7 +21,6 @@
 
 
 pytestmark = [
-    pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domains do not support import."),
     pytest.mark.skipif(
         "/tmp" not in settings.ALLOWED_IMPORT_PATHS,
         reason="Cannot run import-tests unless /tmp is in ALLOWED_IMPORT_PATHS ({}).".format(

pyproject.toml

@@ -35,7 +35,7 @@ dependencies = [
   "Django>=4.2.24,<5.3",  # LTS version, switch only if we have a compelling reason to".
   "django-filter>=23.1,<=25.1",  # Uses CalVer, not released often https://github.com/carltongibson/django-filter
   "django-guid>=3.3.0,<3.6",  # Looks like only bugfixes in z-Stream.
-  "django-import-export>=2.9,<3.4.0",
+  "django-import-export>=4.0,<5.0",
   "django-lifecycle>=1.0,<=1.2.4",
   "djangorestframework>=3.14.0,<=3.16.1",
   "djangorestframework-queryfields>=1.0,<=1.1.0",
@@ -59,7 +59,7 @@ dependencies = [
   "python-gnupg>=0.5.0,<0.6",  # Looks like only bugfixes in z-Stream [changelog only in git]
   "PyYAML>=5.1.1,<6.1",  # Looks like only bugfixes in z-Stream.
   "redis>=4.3.0,<6.5",  # Looks like only bugfixes in z-Stream.
-  "tablib>=3.5.0,<3.6",  # 3.6.0 breaks with import export. Not sure about semver.
+  "tablib>=3.8.0,<4.0",  # 3.6.0 breaks with import export. Not sure about semver.
   "url-normalize>=1.4.3,<2.3",  # SemVer. https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md#changelog
   "uuid6>=2023.5.2,<=2025.0.1",
   "whitenoise>=5.0,<6.12.0",