[PULP-817] Django5 support and adjustments to make imp/exp work with django-imp-exp 4.x.

.github/workflows/scripts/install.sh

@@ -119,7 +119,7 @@ if [ "$TEST" = "azure" ]; then
       - ./azurite:/etc/pulp\
     command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml
   sed -i -e '$a azure_test: true\
-pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": "", "api_root_rewrite_header": "X-API-Root", "content_origin": null, "domain_enabled": true, "rest_framework__default_permission_classes": ["pulpcore.plugin.access_policy.DefaultAccessPolicy"]}\
+pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "api_root_rewrite_header": "X-API-Root", "content_origin": null, "domain_enabled": true, "rest_framework__default_permission_classes": ["pulpcore.plugin.access_policy.DefaultAccessPolicy"]}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
 fi

CHANGES/5324.feature

@@ -0,0 +1 @@
+Adapted PulpImport/Export to allow update django-import-export==4.x.

CHANGES/6988.feature

@@ -0,0 +1,3 @@
+Allow use of Django 5 as well as Django 4. Note the following breaking changes if upgrading to
+Django 5: storage configuration must use the new ``STORAGES`` format instead of
+``DEFAULT_FILE_STORAGE``, Python >= 3.10 is required, and PostgreSQL >= 14 is required.

pulp_file/tests/functional/api/test_download_policies.py

@@ -54,8 +54,7 @@ def test_download_policy(
     download_policy,
 ):
     """Test that "on_demand" and "streamed" download policies work as expected."""
-    settings = pulp_settings
-    if download_policy == "on_demand" and "SFTP" in settings.STORAGES["default"]["BACKEND"]:
+    if download_policy == "on_demand" and "SFTP" in pulp_settings.STORAGES["default"]["BACKEND"]:
         pytest.skip("This storage technology is not properly supported.")
 
     remote = file_remote_ssl_factory(
@@ -150,8 +149,8 @@ def test_download_policy(
     assert expected_checksum == actual_checksum
     if (
         download_policy == "immediate"
-        and settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem"
-        and settings.REDIRECT_TO_OBJECT_STORAGE
+        and pulp_settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem"
+        and pulp_settings.REDIRECT_TO_OBJECT_STORAGE
     ):
         content_disposition = downloaded_file.response_obj.headers.get("Content-Disposition")
         assert content_disposition is not None
@@ -189,7 +188,7 @@ def test_download_policy(
     content_unit = expected_files_list[4]
     content_unit_url = urljoin(distribution_base_url, content_unit[0])
     # The S3 test API project doesn't handle invalid Range values correctly
-    if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
+    if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
         with pytest.raises(ClientResponseError) as exc:
             range_header = {"Range": "bytes=-1-11"}
             download_file(content_unit_url, headers=range_header)

pulpcore/app/importexport.py

@@ -48,6 +48,10 @@ def _write_export(the_tarfile, resource, dest_dir=None):
     # the data in batches to memory and concatenate the json lists via string manipulation.
     with tempfile.NamedTemporaryFile(dir=".", mode="w", encoding="utf8") as temp_file:
         if isinstance(resource.queryset, QuerySet):
+            # If we don't have any of "these" - skip writing
+            if resource.queryset.count() == 0:
+                return
+
             temp_file.write("[")
 
             def process_batch(batch):

pulpcore/app/modelresource.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from import_export import fields
 from import_export.widgets import ForeignKeyWidget
 from logging import getLogger
@@ -36,8 +37,8 @@ def before_import_row(self, row, **kwargs):
         # the export converts None to blank strings but sha384 and sha512 have unique constraints
         # that get triggered if they are blank. convert checksums back into None if they are blank.
         for checksum in ALL_KNOWN_CONTENT_CHECKSUMS:
-            if row[checksum] == "":
-                row[checksum] = None
+            if row[checksum] == "" or checksum not in settings.ALLOWED_CONTENT_CHECKSUMS:
+                del row[checksum]
 
     class Meta:
         model = Artifact

pulpcore/app/tasks/importer.py

@@ -228,6 +228,9 @@ def _import_file(fpath, resource_class, retry=False):
     """
     try:
         log.info(f"Importing file {fpath}.")
+        if not os.path.isfile(fpath):
+            log.info("...empty - skipping.")
+            return []
         with open(fpath, "r") as json_file:
             resource = resource_class()
             log.info(f"...Importing resource {resource.__class__.__name__}.")
@@ -236,6 +239,8 @@ def _import_file(fpath, resource_class, retry=False):
             # overlapping content.
             for batch_str in _impfile_iterator(json_file):
                 data = Dataset().load(StringIO(batch_str))
+                if not data:
+                    return []
                 if retry:
                     curr_attempt = 1
 
@@ -267,6 +272,7 @@ def _import_file(fpath, resource_class, retry=False):
                         try:
                             a_result = resource.import_data(data, raise_errors=True)
                         except Exception as e:  # noqa log on ANY exception and then re-raise
+                            log.error(e)
                             log.error(f"FATAL import-failure importing {fpath}")
                             raise
                 else:

pulpcore/plugin/importexport.py

@@ -41,6 +41,9 @@ def set_up_queryset(self):
     def dehydrate_pulp_domain(self, content):
         return str(content.pulp_domain_id)
 
+    def render(self, value, obj=None, **kwargs):
+        return super().render(value, obj, coerce_to_string=False, **kwargs)
+
     def __init__(self, repo_version=None):
         self.repo_version = repo_version
         if repo_version:

pulpcore/pytest_plugin.py

@@ -608,22 +608,22 @@ def _settings_factory(storage_class=None, storage_settings=None):
         ]
         keys["storages.backends.s3boto3.S3Boto3Storage"] = keys["storages.backends.s3.S3Storage"]
         keys["storages.backends.azure_storage.AzureStorage"] = [
-            "AZURE_ACCOUNT_NAME",
-            "AZURE_CONTAINER",
-            "AZURE_ACCOUNT_KEY",
-            "AZURE_URL_EXPIRATION_SECS",
-            "AZURE_OVERWRITE_FILES",
-            "AZURE_LOCATION",
-            "AZURE_CONNECTION_STRING",
+            "account_name",
+            "azure_container",
+            "account_key",
+            "expiration_secs",
+            "overwrite_files",
+            "location",
+            "connection_string",
         ]
         settings = storage_settings or dict()
         backend = storage_class or pulp_settings.STORAGES["default"]["BACKEND"]
         not_defined_settings = (k for k in keys[backend] if k not in settings)
-        # The CI configures s3 with STORAGES and Azure with legacy
-        # Move all to STORAGES structure on DEFAULT_FILE_STORAGE removal
-        if backend == "storages.backends.s3boto3.S3Boto3Storage":
-            storages_dict = getattr(pulp_settings, "STORAGES", {})
-            storage_options = storages_dict.get("default", {}).get("OPTIONS", {})
+        # Read storage settings from STORAGES.default.OPTIONS when using the default backend
+        default_backend = pulp_settings.STORAGES["default"]["BACKEND"]
+        storages_dict = getattr(pulp_settings, "STORAGES", {})
+        storage_options = storages_dict.get("default", {}).get("OPTIONS", {})
+        if storage_options and backend == default_backend:
             for key in not_defined_settings:
                 settings[key] = storage_options.get(key)
         else:

pulpcore/tests/functional/api/test_artifact_distribution.py

@@ -12,7 +12,6 @@
 
 
 def test_artifact_distribution(random_artifact, pulp_settings):
-    settings = pulp_settings
     artifact_uuid = random_artifact.pulp_href.split("/")[-2]
 
     commands = (
@@ -29,7 +28,7 @@ def test_artifact_distribution(random_artifact, pulp_settings):
     hasher = sha256()
     hasher.update(response.content)
     assert hasher.hexdigest() == random_artifact.sha256
-    if settings.STORAGES["default"]["BACKEND"] in OBJECT_STORAGES:
+    if pulp_settings.STORAGES["default"]["BACKEND"] in OBJECT_STORAGES:
         content_disposition = response.headers.get("Content-Disposition")
         assert content_disposition is not None
         filename = artifact_uuid

pulpcore/tests/functional/api/test_crd_artifacts.py

@@ -143,10 +143,9 @@ def test_upload_mixed_attrs(pulpcore_bindings, pulpcore_random_file):
 def test_delete_artifact(pulpcore_bindings, pulpcore_random_file, gen_user, pulp_settings):
     """Verify that the deletion of artifacts is prohibited for both regular users and
     administrators."""
-    settings = pulp_settings
-    if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem":
+    if pulp_settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem":
         pytest.skip("this test only works for filesystem storage")
-    media_root = settings.MEDIA_ROOT
+    media_root = pulp_settings.MEDIA_ROOT
 
     artifact = pulpcore_bindings.ArtifactsApi.create(str(pulpcore_random_file["name"]))
     path_to_file = os.path.join(media_root, artifact.file)

pulpcore/tests/functional/api/test_crud_domains.py

@@ -52,16 +52,15 @@ def test_crud_domains(pulpcore_bindings, monitor_task):
 @pytest.mark.parallel
 def test_default_domain(pulpcore_bindings, pulp_settings):
     """Test properties around the default domain."""
-    settings = pulp_settings
     domains = pulpcore_bindings.DomainsApi.list(name="default")
     assert domains.count == 1
 
     # Read the default domain, ensure storage is set to default
     default_domain = domains.results[0]
     assert default_domain.name == "default"
-    assert default_domain.storage_class == settings.STORAGES["default"]["BACKEND"]
-    assert default_domain.redirect_to_object_storage == settings.REDIRECT_TO_OBJECT_STORAGE
-    assert default_domain.hide_guarded_distributions == settings.HIDE_GUARDED_DISTRIBUTIONS
+    assert default_domain.storage_class == pulp_settings.STORAGES["default"]["BACKEND"]
+    assert default_domain.redirect_to_object_storage == pulp_settings.REDIRECT_TO_OBJECT_STORAGE
+    assert default_domain.hide_guarded_distributions == pulp_settings.HIDE_GUARDED_DISTRIBUTIONS
 
     # Try to create another default domain
     body = {
@@ -94,8 +93,7 @@ def test_default_domain(pulpcore_bindings, pulp_settings):
 @pytest.mark.parallel
 def test_active_domain_deletion(pulpcore_bindings, monitor_task, pulp_settings):
     """Test trying to delete a domain that is in use, has objects in it."""
-    settings = pulp_settings
-    if not settings.DOMAIN_ENABLED:
+    if not pulp_settings.DOMAIN_ENABLED:
         pytest.skip("Domains not enabled")
     name = str(uuid.uuid4())
     body = {
@@ -137,8 +135,7 @@ def test_orphan_domain_deletion(
     pulp_settings,
 ):
     """Test trying to delete a domain that is in use, has objects in it."""
-    settings = pulp_settings
-    if not settings.DOMAIN_ENABLED:
+    if not pulp_settings.DOMAIN_ENABLED:
         pytest.skip("Domains not enabled")
     body = {
         "name": str(uuid.uuid4()),
@@ -182,8 +179,7 @@ def test_orphan_domain_deletion(
 @pytest.mark.parallel
 def test_special_domain_creation(pulpcore_bindings, gen_object_with_cleanup, pulp_settings):
     """Test many possible domain creation scenarios."""
-    settings = pulp_settings
-    if not settings.DOMAIN_ENABLED:
+    if not pulp_settings.DOMAIN_ENABLED:
         pytest.skip("Domains not enabled")
     # This test needs to account for which environment it is running in
     storage_types = {

pulpcore/tests/functional/api/test_status.py

@@ -126,7 +126,7 @@ def test_storage_per_domain(
     assert default_status.storage != domain_status.storage
 
 
-def verify_get_response(status, expected_schema, settings):
+def verify_get_response(status, expected_schema, pulp_settings):
     """Verify the response to an HTTP GET call.
 
     Verify that several attributes and have the correct type or value.
@@ -140,7 +140,7 @@ def verify_get_response(status, expected_schema, settings):
     assert status["content_settings"]["content_path_prefix"] is not None
 
     assert status["storage"]["used"] is not None
-    if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem":
+    if pulp_settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem":
         assert status["storage"]["free"] is None
         assert status["storage"]["total"] is None
     else:

pulpcore/tests/functional/api/using_plugin/test_crud_repos.py

@@ -299,7 +299,7 @@ def raise_for_invalid_request(remote_attrs):
         """Check if Pulp returns HTTP 400 after issuing an invalid request."""
         with pytest.raises(ApiException) as ae:
             file_bindings.RemotesFileApi.create(remote_attrs)
-            assert ae.value.status == 400
+        assert ae.value.status == 400
 
     # Test the validation of an invalid absolute pathname.
     remote_attrs = {

pulpcore/tests/functional/api/using_plugin/test_orphans.py

@@ -68,12 +68,11 @@ def test_orphans_delete(
     monitor_task,
     pulp_settings,
 ):
-    settings = pulp_settings
     # Verify that the system contains the orphan content unit and the orphan artifact.
     content_unit = file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href)
     artifact = pulpcore_bindings.ArtifactsApi.read(random_artifact.pulp_href)
 
-    if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
+    if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
         # Verify that the artifacts are on disk
         relative_path = pulpcore_bindings.ArtifactsApi.read(content_unit.artifact).file
         artifact_path1 = os.path.join(pulp_settings.MEDIA_ROOT, relative_path)
@@ -89,7 +88,7 @@ def test_orphans_delete(
         with pytest.raises(file_bindings.ApiException) as exc:
             file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href)
         assert exc.value.status == 404
-        if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
+        if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
             assert os.path.exists(artifact_path1) is False
             assert os.path.exists(artifact_path2) is False
 
@@ -102,15 +101,14 @@ def test_orphans_cleanup(
     monitor_task,
     pulp_settings,
 ):
-    settings = pulp_settings
     # Cleanup orphans with a nonzero orphan_protection_time
     monitor_task(pulpcore_bindings.OrphansCleanupApi.cleanup({"orphan_protection_time": 10}).task)
 
     # Verify that the system contains the orphan content unit and the orphan artifact.
     content_unit = file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href)
     artifact = pulpcore_bindings.ArtifactsApi.read(random_artifact.pulp_href)
 
-    if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
+    if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
         # Verify that the artifacts are on disk
         relative_path = pulpcore_bindings.ArtifactsApi.read(content_unit.artifact).file
         artifact_path1 = os.path.join(pulp_settings.MEDIA_ROOT, relative_path)
@@ -125,7 +123,7 @@ def test_orphans_cleanup(
     with pytest.raises(file_bindings.ApiException) as exc:
         file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href)
     assert exc.value.status == 404
-    if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
+    if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
         assert os.path.exists(artifact_path1) is False
         assert os.path.exists(artifact_path2) is False
 

pulpcore/tests/unit/viewsets/test_viewset_base.py

@@ -1,5 +1,5 @@
 import pytest
-from pytest_django.asserts import assertQuerysetEqual
+from pytest_django.asserts import assertQuerySetEqual
 import unittest
 
 from django.http import Http404, QueryDict
@@ -23,7 +23,7 @@ def test_adds_filters():
     queryset = viewset.get_queryset()
     expected = models.RepositoryVersion.objects.filter(repository__pk=repo.pk)
 
-    assertQuerysetEqual(queryset, expected)
+    assertQuerySetEqual(queryset, expected)
 
 
 @pytest.mark.django_db
@@ -38,7 +38,7 @@ def test_does_not_add_filters():
     queryset = viewset.get_queryset()
     expected = models.Repository.objects.all()
 
-    assertQuerysetEqual(queryset, expected)
+    assertQuerySetEqual(queryset, expected)
 
 
 def test_must_define_serializer_class():

pyproject.toml

@@ -35,10 +35,10 @@ dependencies = [
   "backoff>=2.1.2,<2.2.2",
   "click>=8.1.0,<=8.1.8",
   "cryptography>=38.0.1,<45.0.5",
-  "Django~=4.2.0",  # LTS version, switch only if we have a compelling reason to",
+  "Django>=4.2.24,<5.3, !=5.0, !=5.1",  # LTS version, switch only if we have a compelling reason to",
   "django-filter>=23.1,<=25.1",
   "django-guid>=3.3,<=3.5.1",
-  "django-import-export>=2.9,<3.4.0",
+  "django-import-export>=2.9,<5.0",
   "django-lifecycle>=1.0,<=1.2.4",
   "djangorestframework>=3.14.0,<=3.15.2",
   "djangorestframework-queryfields>=1.0,<=1.1.0",
@@ -65,7 +65,7 @@ dependencies = [
   "python-gnupg>=0.5,<=0.5.4",
   "PyYAML>=5.1.1,<=6.0.2",
   "redis>=4.3,<5.2.2",
-  "tablib<3.6.0",
+  "tablib>=3.5.0,<4.0, !=3.6",
   "url-normalize>=1.4.3,<=1.4.3",
   "uuid6>=2023.5.2,<=2024.7.10",
   "whitenoise>=5.0,<6.10.0",

template_config.yml

@@ -1,8 +1,3 @@
-# This config represents the latest values used when running the plugin-template. Any settings that
-# were not present before running plugin-template have been added with their default values.
-
-# generated with plugin_template
-
 api_root: /pulp/
 black: true
 check_commit_message: true
@@ -59,15 +54,20 @@ pulp_settings:
   tmpfile_protection_time: 10
   upload_protection_time: 10
 pulp_settings_azure:
-  AZURE_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
-  AZURE_ACCOUNT_NAME: devstoreaccount1
-  AZURE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;
-  AZURE_CONTAINER: pulp-test
-  AZURE_LOCATION: pulp3
-  AZURE_OVERWRITE_FILES: true
-  AZURE_URL_EXPIRATION_SECS: 120
-  DEFAULT_FILE_STORAGE: storages.backends.azure_storage.AzureStorage
   MEDIA_ROOT: ''
+  STORAGES:
+    default:
+      BACKEND: storages.backends.azure_storage.AzureStorage
+      OPTIONS:
+        account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+        account_name: devstoreaccount1
+        connection_string: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;
+        azure_container: pulp-test
+        location: pulp3
+        overwrite_files: true
+        expiration_secs: 120
+    staticfiles:
+      BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
   api_root_rewrite_header: X-API-Root
   content_origin: null
   domain_enabled: true