[PULP-817] Django5 support and adjustments to make imp/exp work with django-imp-exp 4.x.

.ci/assets/ci_constraints.txt

@@ -16,3 +16,4 @@ azure-storage-blob!=12.28.*
 
 pycares<5
 # older aiodns versions don't pin pycares UB, and are broken by pycares>=5
+

.github/workflows/scripts/install.sh

@@ -79,7 +79,7 @@ cat >> vars/main.yaml << VARSYAML
 pulp_env: {"PULP_CA_BUNDLE": "/etc/pulp/certs/pulp_webserver.crt"}
 pulp_settings: {"allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "orphan_protection_time": 0}
 pulp_scheme: https
-pulp_default_container: ghcr.io/pulp/pulp-ci-centos:latest
+pulp_default_container: ghcr.io/pulp/pulp-ci-centos9:latest
 VARSYAML
 
 SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "generate-bindings" "lowerbounds")
@@ -105,7 +105,7 @@ if [ "$TEST" = "s3" ]; then
   sed -i -e '$a s3_test: true\
 minio_access_key: "'$MINIO_ACCESS_KEY'"\
 minio_secret_key: "'$MINIO_SECRET_KEY'"\
-pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "MEDIA_ROOT": "", "domain_enabled": true, "hide_guarded_distributions": true}\
+pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true, "hide_guarded_distributions": true}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
   export PULP_API_ROOT="/rerouted/djnd/"
@@ -119,7 +119,7 @@ if [ "$TEST" = "azure" ]; then
       - ./azurite:/etc/pulp\
     command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml
   sed -i -e '$a azure_test: true\
-pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": "", "domain_enabled": true}\
+pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
 fi

CHANGES/5324.feature

@@ -0,0 +1 @@
+Adapted PulpImport/Export to allow update django-import-export==4.x.

CHANGES/6988.feature

@@ -0,0 +1 @@
+Allow use of Django5 as well as Django4.

lint_requirements.txt

@@ -6,6 +6,9 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 black==24.3.0
+# Click is pinned because of:
+# https://github.com/pallets/click/issues/3065
+click<8.3
 bump-my-version
 check-manifest
 flake8

pulpcore/app/apps.py

@@ -5,7 +5,6 @@
 from importlib import import_module
 
 from django import apps
-from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.db import connection, transaction
 from django.db.models.signals import post_migrate
@@ -66,6 +65,12 @@ class PulpPluginAppConfig(apps.AppConfig):
 
     def __init__(self, app_name, app_module):
         super().__init__(app_name, app_module)
+        # begin compatibility layer for DEFAULT_FILE_STORAGE deprecation
+        # Workaround for getting the up-to-date settings instance
+        from django.conf import settings
+
+        self.settings = settings
+        # end
 
         try:
             self.version
@@ -307,6 +312,7 @@ def _populate_system_id(sender, apps, verbosity, **kwargs):
 
 
 def _ensure_default_domain(sender, **kwargs):
+    settings = sender.settings
     table_names = connection.introspection.table_names()
     if "core_domain" in table_names:
         from pulpcore.app.util import get_default_domain
@@ -316,11 +322,11 @@ def _ensure_default_domain(sender, **kwargs):
         if (
             settings.HIDE_GUARDED_DISTRIBUTIONS != default.hide_guarded_distributions
             or settings.REDIRECT_TO_OBJECT_STORAGE != default.redirect_to_object_storage
-            or settings.DEFAULT_FILE_STORAGE != default.storage_class
+            or settings.STORAGES["default"]["BACKEND"] != default.storage_class
         ):
             default.hide_guarded_distributions = settings.HIDE_GUARDED_DISTRIBUTIONS
             default.redirect_to_object_storage = settings.REDIRECT_TO_OBJECT_STORAGE
-            default.storage_class = settings.DEFAULT_FILE_STORAGE
+            default.storage_class = settings.STORAGES["default"]["BACKEND"]
             default.save(skip_hooks=True)
 
 
@@ -386,8 +392,9 @@ def _get_permission(perm):
 
 
 def _populate_artifact_serving_distribution(sender, apps, verbosity, **kwargs):
+    settings = sender.settings
     if (
-        settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem"
+        settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem"
         or not settings.REDIRECT_TO_OBJECT_STORAGE
     ):
         try:

pulpcore/app/checks.py

@@ -23,7 +23,7 @@ def content_origin_check(app_configs, **kwargs):
 def storage_paths(app_configs, **kwargs):
     warnings = []
 
-    if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem":
+    if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
         try:
             media_root_dev = Path(settings.MEDIA_ROOT).stat().st_dev
         except OSError:

pulpcore/app/importexport.py

@@ -47,6 +47,10 @@ def _write_export(the_tarfile, resource, dest_dir=None):
     # the data in batches to memory and concatenate the json lists via string manipulation.
     with tempfile.NamedTemporaryFile(dir=".", mode="w", encoding="utf8") as temp_file:
         if isinstance(resource.queryset, QuerySet):
+            # If we don't have any of "these" - skip writing
+            if resource.queryset.count() == 0:
+                return
+
             temp_file.write("[")
 
             def process_batch(batch):
@@ -117,7 +121,7 @@ def export_artifacts(export, artifacts):
     with ProgressReport(**data) as pb:
         pb.BATCH_INTERVAL = 5000
 
-        if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem":
+        if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem":
             with tempfile.TemporaryDirectory(dir=".") as temp_dir:
                 for artifact in pb.iter(artifacts.only("file").iterator()):
                     with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:

pulpcore/app/modelresource.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from import_export import fields
 from import_export.widgets import ForeignKeyWidget
 from logging import getLogger
@@ -36,8 +37,8 @@ def before_import_row(self, row, **kwargs):
         # the export converts None to blank strings but sha384 and sha512 have unique constraints
         # that get triggered if they are blank. convert checksums back into None if they are blank.
         for checksum in ALL_KNOWN_CONTENT_CHECKSUMS:
-            if row[checksum] == "":
-                row[checksum] = None
+            if row[checksum] == "" or checksum not in settings.ALLOWED_CONTENT_CHECKSUMS:
+                del row[checksum]
 
     class Meta:
         model = Artifact

pulpcore/app/models/domain.py

@@ -1,4 +1,5 @@
-from django.core.files.storage import get_storage_class, default_storage
+from django.core.files.storage import default_storage
+from django.utils.module_loading import import_string
 from django.db import models
 from django_lifecycle import hook, BEFORE_DELETE, BEFORE_UPDATE
 
@@ -41,7 +42,7 @@ def get_storage(self):
         """Returns this domain's instantiated storage class."""
         if self.name == "default":
             return default_storage
-        storage_class = get_storage_class(self.storage_class)
+        storage_class = import_string(self.storage_class)
         return storage_class(**self.storage_settings)
 
     @hook(BEFORE_DELETE, when="name", is_now="default")

pulpcore/app/serializers/domain.py

@@ -1,7 +1,7 @@
 from gettext import gettext as _
 
 from django.conf import settings
-from django.core.files.storage import import_string
+from django.utils.module_loading import import_string
 from django.core.exceptions import ImproperlyConfigured
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import extend_schema_field
@@ -40,7 +40,10 @@ def to_representation(self, instance):
         # Should I convert back the saved settings to their Setting names for to_representation?
         if getattr(self.context.get("domain", None), "name", None) == "default":
             for setting_name, field in self.SETTING_MAPPING.items():
-                if value := getattr(settings, setting_name.upper(), None):
+                value = getattr(settings, setting_name, None) or settings.STORAGES["default"].get(
+                    "OPTIONS", {}
+                ).get(field)
+                if value:
                     instance[field] = value
         return super().to_representation(instance)
 

pulpcore/app/settings.py

@@ -17,6 +17,8 @@
 from pkg_resources import iter_entry_points
 
 from cryptography.fernet import Fernet
+from django.conf import global_settings
+from django.core.files.storage import storages  # noqa: F401
 from django.core.exceptions import ImproperlyConfigured
 from django.db import connection
 
@@ -42,7 +44,23 @@
 STATIC_URL = "/assets/"
 STATIC_ROOT = DEPLOY_ROOT / STATIC_URL.strip("/")
 
-DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem"
+# begin compatibility layer for DEFAULT_FILE_STORAGE
+# Django does not allow STORAGES and DEFAULT_FILE_STORAGE in the same settings module
+# (they are mutually exclusive). We set both on global_settings as defaults so that
+# users can override either one via dynaconf, and Django picks the right one for its version.
+_DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem"
+_STORAGES = {
+    "default": {
+        "BACKEND": "pulpcore.app.models.storage.FileSystem",
+    },
+    "staticfiles": {
+        "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+    },
+}
+setattr(global_settings, "DEFAULT_FILE_STORAGE", _DEFAULT_FILE_STORAGE)
+setattr(global_settings, "STORAGES", _STORAGES)
+# end DEFAULT_FILE_STORAGE compatibility layer
+
 REDIRECT_TO_OBJECT_STORAGE = True
 
 WORKING_DIRECTORY = DEPLOY_ROOT / "tmp"
@@ -317,16 +335,18 @@
 from dynaconf import DjangoDynaconf, Validator  # noqa
 
 # Validators
+storage_keys = ("STORAGES.default.BACKEND", "DEFAULT_FILE_STORAGE")
 storage_validator = (
     Validator("REDIRECT_TO_OBJECT_STORAGE", eq=False)
-    | Validator("DEFAULT_FILE_STORAGE", eq="pulpcore.app.models.storage.FileSystem")
-    | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.azure_storage.AzureStorage")
-    | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.s3boto3.S3Boto3Storage")
-    | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.gcloud.GoogleCloudStorage")
+    | Validator(*storage_keys, eq="pulpcore.app.models.storage.FileSystem")
+    | Validator(*storage_keys, eq="storages.backends.azure_storage.AzureStorage")
+    | Validator(*storage_keys, eq="storages.backends.s3boto3.S3Boto3Storage")
+    | Validator(*storage_keys, eq="storages.backends.gcloud.GoogleCloudStorage")
 )
 storage_validator.messages["combined"] = (
-    "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure"
-    "storage backend configured in DEFAULT_FILE_STORAGE."
+    "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure "
+    "storage backend configured in STORAGES['default']['BACKEND'] "
+    "(deprecated DEFAULT_FILE_STORAGE)."
 )
 
 cache_enabled_validator = Validator("CACHE_ENABLED", eq=True)
@@ -473,6 +493,10 @@
     finally:
         connection.close()
 
+# Ensures the cached property storage.backends uses the right value after dynaconf init
+storages._backends = settings.STORAGES.copy()
+storages.backends
+
 settings.set("V3_API_ROOT", settings.API_ROOT + "api/v3/")  # Not user configurable
 settings.set("V3_DOMAIN_API_ROOT", settings.API_ROOT + "<slug:pulp_domain>/api/v3/")
 settings.set("V3_API_ROOT_NO_FRONT_SLASH", settings.V3_API_ROOT.lstrip("/"))

pulpcore/app/tasks/export.py

@@ -71,7 +71,7 @@ def _export_to_file_system(path, relative_paths_to_artifacts, method=FS_EXPORT_M
         ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting
     """
     using_filesystem_storage = (
-        settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem"
+        settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem"
     )
 
     if method != FS_EXPORT_METHODS.WRITE and not using_filesystem_storage:

pulpcore/app/tasks/importer.py

@@ -101,6 +101,9 @@ def _import_file(fpath, resource_class, retry=False):
     """
     try:
         log.info(f"Importing file {fpath}.")
+        if not os.path.isfile(fpath):
+            log.info("...empty - skipping.")
+            return []
         with open(fpath, "r") as json_file:
             resource = resource_class()
             log.info(f"...Importing resource {resource.__class__.__name__}.")
@@ -109,6 +112,8 @@ def _import_file(fpath, resource_class, retry=False):
             # overlapping content.
             for batch_str in _impfile_iterator(json_file):
                 data = Dataset().load(StringIO(batch_str))
+                if not data:
+                    return []
                 if retry:
                     curr_attempt = 1
 
@@ -140,6 +145,7 @@ def _import_file(fpath, resource_class, retry=False):
                         try:
                             a_result = resource.import_data(data, raise_errors=True)
                         except Exception as e:  # noqa log on ANY exception and then re-raise
+                            log.error(e)
                             log.error(f"FATAL import-failure importing {fpath}")
                             raise
                 else:

pulpcore/app/util.py

@@ -387,7 +387,7 @@ def get_artifact_url(artifact, headers=None, http_method=None):
         if settings.DOMAIN_ENABLED:
             loc = f"domain {artifact_domain.name}.storage_class"
         else:
-            loc = "settings.DEFAULT_FILE_STORAGE"
+            loc = "settings.STORAGES['default']['BACKEND']"
 
         raise NotImplementedError(
             f"The value {loc}={artifact_domain.storage_class} does not allow redirecting."
@@ -433,7 +433,9 @@ def get_default_domain():
         try:
             default_domain = Domain.objects.get(name="default")
         except Domain.DoesNotExist:
-            default_domain = Domain(name="default", storage_class=settings.DEFAULT_FILE_STORAGE)
+            default_domain = Domain(
+                name="default", storage_class=settings.STORAGES["default"]["BACKEND"]
+            )
             default_domain.save(skip_hooks=True)
 
     return default_domain

pulpcore/app/views/status.py

@@ -18,7 +18,7 @@
 
 
 def _disk_usage():
-    if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem":
+    if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
         try:
             return shutil.disk_usage(default_storage.location)
         except Exception:

pulpcore/plugin/importexport.py

@@ -41,6 +41,9 @@ def set_up_queryset(self):
     def dehydrate_pulp_domain(self, content):
         return str(content.pulp_domain_id)
 
+    def render(self, value, obj=None, **kwargs):
+        return super().render(value, obj, coerce_to_string=False, **kwargs)
+
     def __init__(self, repo_version=None):
         self.repo_version = repo_version
         if repo_version:

pulpcore/tests/functional/__init__.py

@@ -1076,28 +1076,37 @@ def _domain_factory():
         keys = dict()
         keys["pulpcore.app.models.storage.FileSystem"] = ["MEDIA_ROOT"]
         keys["storages.backends.s3boto3.S3Boto3Storage"] = [
-            "AWS_ACCESS_KEY_ID",
-            "AWS_SECRET_ACCESS_KEY",
-            "AWS_S3_ENDPOINT_URL",
-            "AWS_S3_ADDRESSING_STYLE",
-            "AWS_S3_SIGNATURE_VERSION",
-            "AWS_S3_REGION_NAME",
-            "AWS_STORAGE_BUCKET_NAME",
+            "access_key",
+            "secret_key",
+            "endpoint_url",
+            "addressing_style",
+            "signature_version",
+            "region_name",
+            "bucket_name",
         ]
         keys["storages.backends.azure_storage.AzureStorage"] = [
-            "AZURE_ACCOUNT_NAME",
-            "AZURE_CONTAINER",
-            "AZURE_ACCOUNT_KEY",
-            "AZURE_URL_EXPIRATION_SECS",
-            "AZURE_OVERWRITE_FILES",
-            "AZURE_LOCATION",
+            "account_name",
+            "azure_container",
+            "account_key",
+            "expiration_secs",
+            "overwrite_files",
+            "location",
         ]
         settings = dict()
-        for key in keys[pulp_settings.DEFAULT_FILE_STORAGE]:
-            settings[key] = getattr(pulp_settings, key, None)
+        backend = pulp_settings.STORAGES["default"]["BACKEND"]
+        not_defined_settings = (k for k in keys[backend] if k not in settings)
+        # Read storage settings from STORAGES.default.OPTIONS
+        storages_dict = getattr(pulp_settings, "STORAGES", {})
+        storage_options = storages_dict.get("default", {}).get("OPTIONS", {})
+        if storage_options:
+            for key in not_defined_settings:
+                settings[key] = storage_options.get(key)
+        else:
+            for key in not_defined_settings:
+                settings[key] = getattr(pulp_settings, key, None)
         body = {
             "name": str(uuid.uuid4()),
-            "storage_class": pulp_settings.DEFAULT_FILE_STORAGE,
+            "storage_class": backend,
             "storage_settings": settings,
         }
         return gen_object_with_cleanup(domains_api_client, body)