[PULP-817] Django5 support and adjustments to make imp/exp work with django-imp-exp 4.x.

CHANGES/5324.feature

@@ -0,0 +1 @@
+Adapted PulpImport/Export to allow update django-import-export==4.x.

CHANGES/6988.feature

@@ -0,0 +1 @@
+Allow use of Django5 as well as Django4.

pulpcore/app/importexport.py

@@ -48,6 +48,10 @@ def _write_export(the_tarfile, resource, dest_dir=None):
     # the data in batches to memory and concatenate the json lists via string manipulation.
     with tempfile.NamedTemporaryFile(dir=".", mode="w", encoding="utf8") as temp_file:
         if isinstance(resource.queryset, QuerySet):
+            # If we don't have any of "these" - skip writing
+            if resource.queryset.count() == 0:
+                return
+
             temp_file.write("[")
 
             def process_batch(batch):

pulpcore/app/modelresource.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from import_export import fields
 from import_export.widgets import ForeignKeyWidget
 from logging import getLogger
@@ -36,8 +37,8 @@ def before_import_row(self, row, **kwargs):
         # the export converts None to blank strings but sha384 and sha512 have unique constraints
         # that get triggered if they are blank. convert checksums back into None if they are blank.
         for checksum in ALL_KNOWN_CONTENT_CHECKSUMS:
-            if row[checksum] == "":
-                row[checksum] = None
+            if row[checksum] == "" or checksum not in settings.ALLOWED_CONTENT_CHECKSUMS:
+                del row[checksum]
 
     class Meta:
         model = Artifact

pulpcore/app/tasks/importer.py

@@ -228,6 +228,9 @@ def _import_file(fpath, resource_class, retry=False):
     """
     try:
         log.info(f"Importing file {fpath}.")
+        if not os.path.isfile(fpath):
+            log.info("...empty - skipping.")
+            return []
         with open(fpath, "r") as json_file:
             resource = resource_class()
             log.info(f"...Importing resource {resource.__class__.__name__}.")
@@ -236,6 +239,8 @@ def _import_file(fpath, resource_class, retry=False):
             # overlapping content.
             for batch_str in _impfile_iterator(json_file):
                 data = Dataset().load(StringIO(batch_str))
+                if not data:
+                    return []
                 if retry:
                     curr_attempt = 1
 
@@ -267,6 +272,7 @@ def _import_file(fpath, resource_class, retry=False):
                         try:
                             a_result = resource.import_data(data, raise_errors=True)
                         except Exception as e:  # noqa log on ANY exception and then re-raise
+                            log.error(e)
                             log.error(f"FATAL import-failure importing {fpath}")
                             raise
                 else:

pulpcore/content/handler.py

@@ -7,6 +7,7 @@
 import struct
 from gettext import gettext as _
 from datetime import datetime, timedelta
+from datetime import timezone as dt_timezone
 
 from aiohttp.client_exceptions import ClientResponseError, ClientConnectionError
 from aiohttp.web import FileResponse, StreamResponse, HTTPOk
@@ -439,9 +440,9 @@ def _parse_checkpoint_path(path):
         else:
             raise PathNotResolved(path)
 
-        request_timestamp = request_timestamp.replace(tzinfo=timezone.utc)
+        request_timestamp = request_timestamp.replace(tzinfo=dt_timezone.utc)
         # Future timestamps are not allowed for checkpoints
-        if request_timestamp > datetime.now(tz=timezone.utc):
+        if request_timestamp > datetime.now(tz=dt_timezone.utc):
             raise PathNotResolved(path)
         # The timestamp is truncated to seconds, so we need to cover the whole second
         request_timestamp = request_timestamp.replace(microsecond=999999)

pulpcore/plugin/importexport.py

@@ -41,6 +41,9 @@ def set_up_queryset(self):
     def dehydrate_pulp_domain(self, content):
         return str(content.pulp_domain_id)
 
+    def render(self, value, obj=None, **kwargs):
+        return super().render(value, obj, coerce_to_string=False, **kwargs)
+
     def __init__(self, repo_version=None):
         self.repo_version = repo_version
         if repo_version:

pulpcore/tests/functional/api/using_plugin/test_crud_repos.py

@@ -299,7 +299,7 @@ def raise_for_invalid_request(remote_attrs):
         """Check if Pulp returns HTTP 400 after issuing an invalid request."""
         with pytest.raises(ApiException) as ae:
             file_bindings.RemotesFileApi.create(remote_attrs)
-            assert ae.value.status == 400
+        assert ae.value.status == 400
 
     # Test the validation of an invalid absolute pathname.
     remote_attrs = {

pulpcore/tests/unit/viewsets/test_viewset_base.py

@@ -1,5 +1,5 @@
 import pytest
-from pytest_django.asserts import assertQuerysetEqual
+from pytest_django.asserts import assertQuerySetEqual
 import unittest
 
 from django.http import Http404, QueryDict
@@ -23,7 +23,7 @@ def test_adds_filters():
     queryset = viewset.get_queryset()
     expected = models.RepositoryVersion.objects.filter(repository__pk=repo.pk)
 
-    assertQuerysetEqual(queryset, expected)
+    assertQuerySetEqual(queryset, expected)
 
 
 @pytest.mark.django_db
@@ -38,7 +38,7 @@ def test_does_not_add_filters():
     queryset = viewset.get_queryset()
     expected = models.Repository.objects.all()
 
-    assertQuerysetEqual(queryset, expected)
+    assertQuerySetEqual(queryset, expected)
 
 
 def test_must_define_serializer_class():

pyproject.toml

@@ -32,10 +32,10 @@ dependencies = [
   "backoff>=2.1.2,<2.3",  # Looks like only bugfixes in z-Stream.
   "click>=8.1.0,<8.3",  # Uses milestone.feature.fix https://palletsprojects.com/versions .
   "cryptography>=44.0.3,<46.0",  # SemVer compatible https://cryptography.io/en/latest/api-stability/#versioning .
-  "Django~=4.2.0",  # LTS version, switch only if we have a compelling reason to".
+  "Django>=4.2.24,<5.3, !=5.0, !=5.1",  # LTS version, switch only if we have a compelling reason to".
   "django-filter>=23.1,<=25.1",  # Looks like CalVer.
   "django-guid>=3.3.0,<3.6",  # Looks like only bugfixes in z-Stream.
-  "django-import-export>=2.9,<3.4.0",
+  "django-import-export>=2.9,<5.0",
   "django-lifecycle>=1.0,<=1.2.4",
   "djangorestframework>=3.14.0,<=3.16.1",
   "djangorestframework-queryfields>=1.0,<=1.1.0",
@@ -59,7 +59,7 @@ dependencies = [
   "python-gnupg>=0.5,<=0.5.4",
   "PyYAML>=5.1.1,<6.1",  # Looks like only bugfixes in z-Stream.
   "redis>=4.3.0,<6.5",  # Looks like only bugfixes in z-Stream.
-  "tablib>=3.5.0,<3.6",  # 3.6.0 breaks with import export. Not sure about semver.
+  "tablib>=3.5.0,<4.0, !=3.6",  # 3.6.0 breaks with import export. Not sure about semver.
   "url-normalize>=1.4.3,<2.3",  # SemVer. https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md#changelog
   "uuid6>=2023.5.2,<=2025.0.1",
   "whitenoise>=5.0,<6.10.0",