From 26b949e36f3adb69f0dffd0d083f53802f7642ea Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Fri, 6 Feb 2026 18:27:50 -0500 Subject: [PATCH] [PULP-817] Django5 support and adjustments to make imp/exp work with django-imp-exp 4.x. (cherry picked from commit 793c0ad638254ef76fb379e44d49a5dea0613751) closes #5324. closes #6988. Co-authored-by: Grant Gainey Assisted by: cursor agent Co-authored-by: Cursor --- .github/workflows/scripts/install.sh | 4 +- CHANGES/5324.feature | 1 + CHANGES/6988.feature | 3 ++ .../functional/api/test_download_policies.py | 9 ++-- pulpcore/app/apps.py | 17 +++++-- pulpcore/app/checks.py | 2 +- pulpcore/app/importexport.py | 6 ++- pulpcore/app/modelresource.py | 5 +- pulpcore/app/models/domain.py | 5 +- pulpcore/app/serializers/domain.py | 5 +- pulpcore/app/settings.py | 38 +++++++++++--- pulpcore/app/tasks/export.py | 2 +- pulpcore/app/tasks/importer.py | 6 +++ pulpcore/app/util.py | 6 ++- pulpcore/plugin/importexport.py | 3 ++ pulpcore/tests/functional/__init__.py | 43 +++++++++------- .../api/test_artifact_distribution.py | 7 +-- .../functional/api/test_crd_artifacts.py | 7 ++- .../tests/functional/api/test_crud_domains.py | 23 +++++---- pulpcore/tests/functional/api/test_status.py | 13 +++-- .../api/using_plugin/test_orphans.py | 22 ++++---- pulpcore/tests/unit/models/test_content.py | 2 +- .../tests/unit/viewsets/test_viewset_base.py | 6 +-- requirements.txt | 6 +-- template_config.yml | 50 ++++++++++--------- 25 files changed, 178 insertions(+), 113 deletions(-) create mode 100644 CHANGES/5324.feature create mode 100644 CHANGES/6988.feature diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 82a9209249b..3a72c0fbb1b 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -105,7 +105,7 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "MEDIA_ROOT": "", "authentication_backends": "@merge django.contrib.auth.backends.RemoteUserBackend", "authentication_json_header": "HTTP_X_RH_IDENTITY", "authentication_json_header_jq_filter": ".identity.user.username", "domain_enabled": true, "hide_guarded_distributions": true, "rest_framework__default_authentication_classes": "@merge pulpcore.app.authentication.JSONHeaderRemoteAuthentication"}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "authentication_backends": "@merge django.contrib.auth.backends.RemoteUserBackend", "authentication_json_header": "HTTP_X_RH_IDENTITY", "authentication_json_header_jq_filter": ".identity.user.username", "domain_enabled": true, "hide_guarded_distributions": true, "rest_framework__default_authentication_classes": "@merge pulpcore.app.authentication.JSONHeaderRemoteAuthentication"}\ pulp_scenario_env: {}\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" @@ -119,7 +119,7 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": "", "domain_enabled": true}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true}\ pulp_scenario_env: {}\ ' vars/main.yaml fi diff --git a/CHANGES/5324.feature b/CHANGES/5324.feature new file mode 100644 index 00000000000..d2d6cc140ce --- /dev/null +++ b/CHANGES/5324.feature @@ -0,0 +1 @@ +Adapted PulpImport/Export to allow update django-import-export==4.x. \ No newline at end of file diff --git a/CHANGES/6988.feature b/CHANGES/6988.feature new file mode 100644 index 00000000000..11200bdaca7 --- /dev/null +++ b/CHANGES/6988.feature @@ -0,0 +1,3 @@ +Allow use of Django 5 as well as Django 4. Note the following breaking changes if upgrading to +Django 5: storage configuration must use the new ``STORAGES`` format instead of +``DEFAULT_FILE_STORAGE``, Python >= 3.10 is required, and PostgreSQL >= 14 is required. diff --git a/pulp_file/tests/functional/api/test_download_policies.py b/pulp_file/tests/functional/api/test_download_policies.py index 08bd76c4f25..0d13ff66cff 100644 --- a/pulp_file/tests/functional/api/test_download_policies.py +++ b/pulp_file/tests/functional/api/test_download_policies.py @@ -11,7 +11,6 @@ from pulpcore.tests.functional.utils import get_files_in_manifest, download_file -from pulpcore.app import settings from pulpcore.client.pulp_file import FileFilePublication, RepositorySyncURL @@ -59,7 +58,7 @@ def test_download_policy( download_policy, ): """Test that "on_demand" and "streamed" download policies work as expected.""" - if download_policy == "on_demand" and "SFTP" in pulp_settings.DEFAULT_FILE_STORAGE: + if download_policy == "on_demand" and "SFTP" in pulp_settings.STORAGES["default"]["BACKEND"]: pytest.skip("This storage technology is not properly supported.") remote = file_remote_ssl_factory( @@ -153,8 +152,8 @@ def test_download_policy( assert expected_checksum == actual_checksum if ( download_policy == "immediate" - and settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem" - and settings.REDIRECT_TO_OBJECT_STORAGE + and pulp_settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem" + and pulp_settings.REDIRECT_TO_OBJECT_STORAGE ): content_disposition = downloaded_file.response_obj.headers.get("Content-Disposition") assert content_disposition is not None @@ -192,7 +191,7 @@ def test_download_policy( content_unit = expected_files_list[4] content_unit_url = urljoin(distribution.base_url, content_unit[0]) # The S3 test API project doesn't handle invalid Range values correctly - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": with pytest.raises(ClientResponseError) as exc: range_header = {"Range": "bytes=-1-11"} download_file(content_unit_url, headers=range_header) diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index efc6b42b1ce..7bf7cc87788 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -5,7 +5,6 @@ from importlib import import_module from django import apps -from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.db import connection, transaction from django.db.models.signals import post_migrate, pre_migrate @@ -68,6 +67,12 @@ class PulpPluginAppConfig(apps.AppConfig): def __init__(self, app_name, app_module): super().__init__(app_name, app_module) + # begin compatibility layer for DEFAULT_FILE_STORAGE deprecation + # Workaround for getting the up-to-date settings instance + from django.conf import settings + + self.settings = settings + # end try: self.version @@ -268,6 +273,8 @@ def _clean_app_status(sender, apps, verbosity, **kwargs): from django.contrib.postgres.functions import TransactionNow from django.utils import timezone + settings = sender.settings + app_ttl_map = [ ("ApiAppStatus", settings.API_APP_TTL), ("ContentAppStatus", settings.CONTENT_APP_TTL), @@ -340,6 +347,7 @@ def _populate_system_id(sender, apps, verbosity, **kwargs): def _ensure_default_domain(sender, **kwargs): + settings = sender.settings table_names = connection.introspection.table_names() if "core_domain" in table_names: from pulpcore.app.util import get_default_domain @@ -349,11 +357,11 @@ def _ensure_default_domain(sender, **kwargs): if ( settings.HIDE_GUARDED_DISTRIBUTIONS != default.hide_guarded_distributions or settings.REDIRECT_TO_OBJECT_STORAGE != default.redirect_to_object_storage - or settings.DEFAULT_FILE_STORAGE != default.storage_class + or settings.STORAGES["default"]["BACKEND"] != default.storage_class ): default.hide_guarded_distributions = settings.HIDE_GUARDED_DISTRIBUTIONS default.redirect_to_object_storage = settings.REDIRECT_TO_OBJECT_STORAGE - default.storage_class = settings.DEFAULT_FILE_STORAGE + default.storage_class = settings.STORAGES["default"]["BACKEND"] default.save(skip_hooks=True) @@ -419,8 +427,9 @@ def _get_permission(perm): def _populate_artifact_serving_distribution(sender, apps, verbosity, **kwargs): + settings = sender.settings if ( - settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem" + settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem" or not settings.REDIRECT_TO_OBJECT_STORAGE ): try: diff --git a/pulpcore/app/checks.py b/pulpcore/app/checks.py index 0cd4a0478b0..ec7c405ef80 100644 --- a/pulpcore/app/checks.py +++ b/pulpcore/app/checks.py @@ -23,7 +23,7 @@ def content_origin_check(app_configs, **kwargs): def storage_paths(app_configs, **kwargs): warnings = [] - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": try: media_root_dev = Path(settings.MEDIA_ROOT).stat().st_dev except OSError: diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py index 3dab3aa6ee9..dd30042b134 100644 --- a/pulpcore/app/importexport.py +++ b/pulpcore/app/importexport.py @@ -48,6 +48,10 @@ def _write_export(the_tarfile, resource, dest_dir=None): # the data in batches to memory and concatenate the json lists via string manipulation. with tempfile.NamedTemporaryFile(dir=".", mode="w", encoding="utf8") as temp_file: if isinstance(resource.queryset, QuerySet): + # If we don't have any of "these" - skip writing + if resource.queryset.count() == 0: + return + temp_file.write("[") def process_batch(batch): @@ -114,7 +118,7 @@ def export_artifacts(export, artifact_pks): with ProgressReport(**data) as pb: pb.BATCH_INTERVAL = 5000 - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": with tempfile.TemporaryDirectory(dir=".") as temp_dir: for offset in range(0, len(artifact_pks), EXPORT_BATCH_SIZE): batch = artifact_pks[offset : offset + EXPORT_BATCH_SIZE] diff --git a/pulpcore/app/modelresource.py b/pulpcore/app/modelresource.py index 54d3522fa98..632e982f06a 100644 --- a/pulpcore/app/modelresource.py +++ b/pulpcore/app/modelresource.py @@ -1,3 +1,4 @@ +from django.conf import settings from import_export import fields from import_export.widgets import ForeignKeyWidget from logging import getLogger @@ -36,8 +37,8 @@ def before_import_row(self, row, **kwargs): # the export converts None to blank strings but sha384 and sha512 have unique constraints # that get triggered if they are blank. convert checksums back into None if they are blank. for checksum in ALL_KNOWN_CONTENT_CHECKSUMS: - if row[checksum] == "": - row[checksum] = None + if row[checksum] == "" or checksum not in settings.ALLOWED_CONTENT_CHECKSUMS: + del row[checksum] class Meta: model = Artifact diff --git a/pulpcore/app/models/domain.py b/pulpcore/app/models/domain.py index fd0bbdff115..407b58f3729 100644 --- a/pulpcore/app/models/domain.py +++ b/pulpcore/app/models/domain.py @@ -1,6 +1,7 @@ from opentelemetry.metrics import Observation -from django.core.files.storage import get_storage_class, default_storage +from django.core.files.storage import default_storage +from django.utils.module_loading import import_string from django.db import models from django_lifecycle import hook, BEFORE_DELETE, BEFORE_UPDATE, AFTER_CREATE @@ -44,7 +45,7 @@ def get_storage(self): """Returns this domain's instantiated storage class.""" if self.name == "default": return default_storage - storage_class = get_storage_class(self.storage_class) + storage_class = import_string(self.storage_class) return storage_class(**self.storage_settings) @hook(BEFORE_DELETE, when="name", is_now="default") diff --git a/pulpcore/app/serializers/domain.py b/pulpcore/app/serializers/domain.py index 4348081066a..1e2a68d8331 100644 --- a/pulpcore/app/serializers/domain.py +++ b/pulpcore/app/serializers/domain.py @@ -40,7 +40,10 @@ def to_representation(self, instance): # Should I convert back the saved settings to their Setting names for to_representation? if getattr(self.context.get("domain", None), "name", None) == "default": for setting_name, field in self.SETTING_MAPPING.items(): - if value := getattr(settings, setting_name.upper(), None): + value = getattr(settings, setting_name, None) or settings.STORAGES["default"].get( + "OPTIONS", {} + ).get(field) + if value: instance[field] = value return super().to_representation(instance) diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py index de87d7599df..22a3b525f1f 100644 --- a/pulpcore/app/settings.py +++ b/pulpcore/app/settings.py @@ -16,6 +16,8 @@ from pathlib import Path from cryptography.fernet import Fernet +from django.conf import global_settings +from django.core.files.storage import storages # noqa: F401 from django.core.exceptions import ImproperlyConfigured from django.db import connection @@ -48,7 +50,23 @@ STATIC_URL = "/assets/" STATIC_ROOT = DEPLOY_ROOT / STATIC_URL.strip("/") -DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem" +# begin compatibility layer for DEFAULT_FILE_STORAGE +# Django does not allow STORAGES and DEFAULT_FILE_STORAGE in the same settings module +# (they are mutually exclusive). We set both on global_settings as defaults so that +# users can override either one via dynaconf, and Django picks the right one for its version. +_DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem" +_STORAGES = { + "default": { + "BACKEND": "pulpcore.app.models.storage.FileSystem", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, +} +setattr(global_settings, "DEFAULT_FILE_STORAGE", _DEFAULT_FILE_STORAGE) +setattr(global_settings, "STORAGES", _STORAGES) +# end DEFAULT_FILE_STORAGE compatibility layer + REDIRECT_TO_OBJECT_STORAGE = True WORKING_DIRECTORY = DEPLOY_ROOT / "tmp" @@ -333,16 +351,18 @@ from dynaconf import DjangoDynaconf, Validator # noqa # Validators +storage_keys = ("STORAGES.default.BACKEND", "DEFAULT_FILE_STORAGE") storage_validator = ( Validator("REDIRECT_TO_OBJECT_STORAGE", eq=False) - | Validator("DEFAULT_FILE_STORAGE", eq="pulpcore.app.models.storage.FileSystem") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.azure_storage.AzureStorage") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.s3boto3.S3Boto3Storage") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.gcloud.GoogleCloudStorage") + | Validator(*storage_keys, eq="pulpcore.app.models.storage.FileSystem") + | Validator(*storage_keys, eq="storages.backends.azure_storage.AzureStorage") + | Validator(*storage_keys, eq="storages.backends.s3boto3.S3Boto3Storage") + | Validator(*storage_keys, eq="storages.backends.gcloud.GoogleCloudStorage") ) storage_validator.messages["combined"] = ( - "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure" - "storage backend configured in DEFAULT_FILE_STORAGE." + "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure " + "storage backend configured in STORAGES['default']['BACKEND'] " + "(deprecated DEFAULT_FILE_STORAGE)." ) cache_enabled_validator = Validator("CACHE_ENABLED", eq=True) @@ -514,6 +534,10 @@ finally: connection.close() +# Ensures the cached property storage.backends uses the right value after dynaconf init +storages._backends = settings.STORAGES.copy() +storages.backends + settings.set("V3_API_ROOT", settings.API_ROOT + "api/v3/") # Not user configurable settings.set("V3_DOMAIN_API_ROOT", settings.API_ROOT + "/api/v3/") settings.set("V3_API_ROOT_NO_FRONT_SLASH", settings.V3_API_ROOT.lstrip("/")) diff --git a/pulpcore/app/tasks/export.py b/pulpcore/app/tasks/export.py index 71956d2359f..e7ef5f164af 100644 --- a/pulpcore/app/tasks/export.py +++ b/pulpcore/app/tasks/export.py @@ -70,7 +70,7 @@ def _export_to_file_system(path, relative_paths_to_artifacts, method=FS_EXPORT_M ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting """ using_filesystem_storage = ( - settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem" + settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem" ) if method != FS_EXPORT_METHODS.WRITE and not using_filesystem_storage: diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py index d26316fbeb2..7eb67fc8059 100644 --- a/pulpcore/app/tasks/importer.py +++ b/pulpcore/app/tasks/importer.py @@ -228,6 +228,9 @@ def _import_file(fpath, resource_class, retry=False): """ try: log.info(f"Importing file {fpath}.") + if not os.path.isfile(fpath): + log.info("...empty - skipping.") + return [] with open(fpath, "r") as json_file: resource = resource_class() log.info(f"...Importing resource {resource.__class__.__name__}.") @@ -236,6 +239,8 @@ def _import_file(fpath, resource_class, retry=False): # overlapping content. for batch_str in _impfile_iterator(json_file): data = Dataset().load(StringIO(batch_str)) + if not data: + return [] if retry: curr_attempt = 1 @@ -267,6 +272,7 @@ def _import_file(fpath, resource_class, retry=False): try: a_result = resource.import_data(data, raise_errors=True) except Exception as e: # noqa log on ANY exception and then re-raise + log.error(e) log.error(f"FATAL import-failure importing {fpath}") raise else: diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py index 96f214509c9..09b5a73c92d 100644 --- a/pulpcore/app/util.py +++ b/pulpcore/app/util.py @@ -413,7 +413,7 @@ def get_artifact_url(artifact, headers=None, http_method=None): if settings.DOMAIN_ENABLED: loc = f"domain {artifact_domain.name}.storage_class" else: - loc = "settings.DEFAULT_FILE_STORAGE" + loc = "settings.STORAGES['default']['BACKEND']" raise NotImplementedError( f"The value {loc}={artifact_domain.storage_class} does not allow redirecting." @@ -459,7 +459,9 @@ def get_default_domain(): try: default_domain = Domain.objects.get(name="default") except Domain.DoesNotExist: - default_domain = Domain(name="default", storage_class=settings.DEFAULT_FILE_STORAGE) + default_domain = Domain( + name="default", storage_class=settings.STORAGES["default"]["BACKEND"] + ) default_domain.save(skip_hooks=True) return default_domain diff --git a/pulpcore/plugin/importexport.py b/pulpcore/plugin/importexport.py index 747a7c0cffa..7ca48e175bc 100644 --- a/pulpcore/plugin/importexport.py +++ b/pulpcore/plugin/importexport.py @@ -41,6 +41,9 @@ def set_up_queryset(self): def dehydrate_pulp_domain(self, content): return str(content.pulp_domain_id) + def render(self, value, obj=None, **kwargs): + return super().render(value, obj, coerce_to_string=False, **kwargs) + def __init__(self, repo_version=None): self.repo_version = repo_version if repo_version: diff --git a/pulpcore/tests/functional/__init__.py b/pulpcore/tests/functional/__init__.py index 0f2960e9aaa..434b1198ef6 100644 --- a/pulpcore/tests/functional/__init__.py +++ b/pulpcore/tests/functional/__init__.py @@ -963,29 +963,38 @@ def _domain_factory(): keys = dict() keys["pulpcore.app.models.storage.FileSystem"] = ["MEDIA_ROOT"] keys["storages.backends.s3boto3.S3Boto3Storage"] = [ - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "AWS_S3_ENDPOINT_URL", - "AWS_S3_ADDRESSING_STYLE", - "AWS_S3_SIGNATURE_VERSION", - "AWS_S3_REGION_NAME", - "AWS_STORAGE_BUCKET_NAME", + "access_key", + "secret_key", + "endpoint_url", + "addressing_style", + "signature_version", + "region_name", + "bucket_name", ] keys["storages.backends.azure_storage.AzureStorage"] = [ - "AZURE_ACCOUNT_NAME", - "AZURE_CONTAINER", - "AZURE_ACCOUNT_KEY", - "AZURE_URL_EXPIRATION_SECS", - "AZURE_OVERWRITE_FILES", - "AZURE_LOCATION", - "AZURE_CONNECTION_STRING", + "account_name", + "azure_container", + "account_key", + "expiration_secs", + "overwrite_files", + "location", + "connection_string", ] settings = dict() - for key in keys[pulp_settings.DEFAULT_FILE_STORAGE]: - settings[key] = getattr(pulp_settings, key, None) + backend = pulp_settings.STORAGES["default"]["BACKEND"] + not_defined_settings = (k for k in keys[backend] if k not in settings) + # Read storage settings from STORAGES.default.OPTIONS + storages_dict = getattr(pulp_settings, "STORAGES", {}) + storage_options = storages_dict.get("default", {}).get("OPTIONS", {}) + if storage_options: + for key in not_defined_settings: + settings[key] = storage_options.get(key) + else: + for key in not_defined_settings: + settings[key] = getattr(pulp_settings, key, None) body = { "name": str(uuid.uuid4()), - "storage_class": pulp_settings.DEFAULT_FILE_STORAGE, + "storage_class": backend, "storage_settings": settings, } return gen_object_with_cleanup(pulpcore_bindings.DomainsApi, body) diff --git a/pulpcore/tests/functional/api/test_artifact_distribution.py b/pulpcore/tests/functional/api/test_artifact_distribution.py index d6735347c22..bdcaaf9cf32 100644 --- a/pulpcore/tests/functional/api/test_artifact_distribution.py +++ b/pulpcore/tests/functional/api/test_artifact_distribution.py @@ -2,9 +2,6 @@ import subprocess from hashlib import sha256 -from django.conf import settings - - OBJECT_STORAGES = ( "storages.backends.s3boto3.S3Boto3Storage", "storages.backends.azure_storage.AzureStorage", @@ -12,7 +9,7 @@ ) -def test_artifact_distribution(random_artifact): +def test_artifact_distribution(random_artifact, pulp_settings): artifact_uuid = random_artifact.pulp_href.split("/")[-2] commands = ( @@ -29,7 +26,7 @@ def test_artifact_distribution(random_artifact): hasher = sha256() hasher.update(response.content) assert hasher.hexdigest() == random_artifact.sha256 - if settings.DEFAULT_FILE_STORAGE in OBJECT_STORAGES: + if pulp_settings.STORAGES["default"]["BACKEND"] in OBJECT_STORAGES: content_disposition = response.headers.get("Content-Disposition") assert content_disposition is not None filename = artifact_uuid diff --git a/pulpcore/tests/functional/api/test_crd_artifacts.py b/pulpcore/tests/functional/api/test_crd_artifacts.py index 8b099979fcb..ce08ec4e03f 100644 --- a/pulpcore/tests/functional/api/test_crd_artifacts.py +++ b/pulpcore/tests/functional/api/test_crd_artifacts.py @@ -6,7 +6,6 @@ import uuid import pytest -from django.conf import settings from pulpcore.client.pulpcore import ApiException @@ -144,11 +143,11 @@ def test_upload_mixed_attrs(artifacts_api_client, pulpcore_random_file): @pytest.mark.parallel -def test_delete_artifact(artifacts_api_client, pulpcore_random_file): +def test_delete_artifact(artifacts_api_client, pulpcore_random_file, pulp_settings): """Delete an artifact, it is removed from the filesystem.""" - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": pytest.skip("this test only works for filesystem storage") - media_root = settings.MEDIA_ROOT + media_root = pulp_settings.MEDIA_ROOT artifact = artifacts_api_client.create(pulpcore_random_file["name"]) path_to_file = os.path.join(media_root, artifact.file) diff --git a/pulpcore/tests/functional/api/test_crud_domains.py b/pulpcore/tests/functional/api/test_crud_domains.py index 9d9caa84c38..653935052af 100644 --- a/pulpcore/tests/functional/api/test_crud_domains.py +++ b/pulpcore/tests/functional/api/test_crud_domains.py @@ -5,8 +5,6 @@ import string import json from pulpcore.client.pulpcore import ApiException -from pulpcore.app import settings - from pulpcore.tests.functional.utils import PulpTaskError @@ -51,7 +49,7 @@ def test_crud_domains(domains_api_client, monitor_task): @pytest.mark.parallel -def test_default_domain(domains_api_client): +def test_default_domain(domains_api_client, pulp_settings): """Test properties around the default domain.""" domains = domains_api_client.list(name="default") assert domains.count == 1 @@ -59,9 +57,9 @@ def test_default_domain(domains_api_client): # Read the default domain, ensure storage is set to default default_domain = domains.results[0] assert default_domain.name == "default" - assert default_domain.storage_class == settings.DEFAULT_FILE_STORAGE - assert default_domain.redirect_to_object_storage == settings.REDIRECT_TO_OBJECT_STORAGE - assert default_domain.hide_guarded_distributions == settings.HIDE_GUARDED_DISTRIBUTIONS + assert default_domain.storage_class == pulp_settings.STORAGES["default"]["BACKEND"] + assert default_domain.redirect_to_object_storage == pulp_settings.REDIRECT_TO_OBJECT_STORAGE + assert default_domain.hide_guarded_distributions == pulp_settings.HIDE_GUARDED_DISTRIBUTIONS # Try to create another default domain body = { @@ -92,9 +90,11 @@ def test_default_domain(domains_api_client): @pytest.mark.parallel -def test_active_domain_deletion(domains_api_client, rbac_contentguard_api_client, monitor_task): +def test_active_domain_deletion( + domains_api_client, rbac_contentguard_api_client, monitor_task, pulp_settings +): """Test trying to delete a domain that is in use, has objects in it.""" - if not settings.DOMAIN_ENABLED: + if not pulp_settings.DOMAIN_ENABLED: pytest.skip("Domains not enabled") name = str(uuid.uuid4()) body = { @@ -134,9 +134,10 @@ def test_orphan_domain_deletion( gen_object_with_cleanup, monitor_task, tmp_path, + pulp_settings, ): """Test trying to delete a domain that is in use, has objects in it.""" - if not settings.DOMAIN_ENABLED: + if not pulp_settings.DOMAIN_ENABLED: pytest.skip("Domains not enabled") body = { "name": str(uuid.uuid4()), @@ -178,9 +179,9 @@ def test_orphan_domain_deletion( @pytest.mark.parallel -def test_special_domain_creation(domains_api_client, gen_object_with_cleanup): +def test_special_domain_creation(domains_api_client, gen_object_with_cleanup, pulp_settings): """Test many possible domain creation scenarios.""" - if not settings.DOMAIN_ENABLED: + if not pulp_settings.DOMAIN_ENABLED: pytest.skip("Domains not enabled") # This test needs to account for which environment it is running in storage_types = { diff --git a/pulpcore/tests/functional/api/test_status.py b/pulpcore/tests/functional/api/test_status.py index 10b372b5695..5a5517cb8e3 100644 --- a/pulpcore/tests/functional/api/test_status.py +++ b/pulpcore/tests/functional/api/test_status.py @@ -2,7 +2,6 @@ import pytest -from django.conf import settings from jsonschema import validate from pulpcore.client.pulpcore import ApiException @@ -59,24 +58,24 @@ @pytest.mark.parallel -def test_get_authenticated(test_path, pulpcore_bindings): +def test_get_authenticated(test_path, pulpcore_bindings, pulp_settings): """GET the status path with valid credentials. Verify the response with :meth:`verify_get_response`. """ response = pulpcore_bindings.StatusApi.status_read() - verify_get_response(response.to_dict(), STATUS) + verify_get_response(response.to_dict(), STATUS, pulp_settings) @pytest.mark.parallel -def test_get_unauthenticated(test_path, pulpcore_bindings, anonymous_user): +def test_get_unauthenticated(test_path, pulpcore_bindings, anonymous_user, pulp_settings): """GET the status path with no credentials. Verify the response with :meth:`verify_get_response`. """ with anonymous_user: response = pulpcore_bindings.StatusApi.status_read() - verify_get_response(response.to_dict(), STATUS) + verify_get_response(response.to_dict(), STATUS, pulp_settings) @pytest.mark.parallel @@ -129,7 +128,7 @@ def test_storage_per_domain( assert default_status.storage != domain_status.storage -def verify_get_response(status, expected_schema): +def verify_get_response(status, expected_schema, pulp_settings): """Verify the response to an HTTP GET call. Verify that several attributes and have the correct type or value. @@ -144,7 +143,7 @@ def verify_get_response(status, expected_schema): assert status["content_settings"]["content_path_prefix"] is not None assert status["storage"]["used"] is not None - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": assert status["storage"]["free"] is None assert status["storage"]["total"] is None else: diff --git a/pulpcore/tests/functional/api/using_plugin/test_orphans.py b/pulpcore/tests/functional/api/using_plugin/test_orphans.py index 68f4d9fd871..1064ecf6245 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_orphans.py +++ b/pulpcore/tests/functional/api/using_plugin/test_orphans.py @@ -3,8 +3,6 @@ import os import pytest -from pulpcore.app import settings - def test_content_orphan_filter( file_content_unit_with_name_factory, @@ -71,16 +69,17 @@ def test_orphans_delete( pulpcore_bindings, file_bindings, monitor_task, + pulp_settings, ): # Verify that the system contains the orphan content unit and the orphan artifact. content_unit = file_content_api_client.read(file_random_content_unit.pulp_href) artifact = artifacts_api_client.read(random_artifact.pulp_href) - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": # Verify that the artifacts are on disk relative_path = artifacts_api_client.read(content_unit.artifact).file - artifact_path1 = os.path.join(settings.MEDIA_ROOT, relative_path) - artifact_path2 = os.path.join(settings.MEDIA_ROOT, artifact.file) + artifact_path1 = os.path.join(pulp_settings.MEDIA_ROOT, relative_path) + artifact_path2 = os.path.join(pulp_settings.MEDIA_ROOT, artifact.file) assert os.path.exists(artifact_path1) is True assert os.path.exists(artifact_path2) is True @@ -88,11 +87,11 @@ def test_orphans_delete( monitor_task(pulpcore_bindings.OrphansApi.delete().task) # Assert that the content unit and artifact are gone - if settings.ORPHAN_PROTECTION_TIME == 0: + if pulp_settings.ORPHAN_PROTECTION_TIME == 0: with pytest.raises(file_bindings.ApiException) as exc: file_content_api_client.read(file_random_content_unit.pulp_href) assert exc.value.status == 404 - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": assert os.path.exists(artifact_path1) is False assert os.path.exists(artifact_path2) is False @@ -105,6 +104,7 @@ def test_orphans_cleanup( pulpcore_bindings, file_bindings, monitor_task, + pulp_settings, ): # Cleanup orphans with a nonzero orphan_protection_time monitor_task(pulpcore_bindings.OrphansCleanupApi.cleanup({"orphan_protection_time": 10}).task) @@ -113,11 +113,11 @@ def test_orphans_cleanup( content_unit = file_content_api_client.read(file_random_content_unit.pulp_href) artifact = artifacts_api_client.read(random_artifact.pulp_href) - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": # Verify that the artifacts are on disk relative_path = artifacts_api_client.read(content_unit.artifact).file - artifact_path1 = os.path.join(settings.MEDIA_ROOT, relative_path) - artifact_path2 = os.path.join(settings.MEDIA_ROOT, artifact.file) + artifact_path1 = os.path.join(pulp_settings.MEDIA_ROOT, relative_path) + artifact_path2 = os.path.join(pulp_settings.MEDIA_ROOT, artifact.file) assert os.path.exists(artifact_path1) is True assert os.path.exists(artifact_path2) is True @@ -128,7 +128,7 @@ def test_orphans_cleanup( with pytest.raises(file_bindings.ApiException) as exc: file_content_api_client.read(file_random_content_unit.pulp_href) assert exc.value.status == 404 - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": assert os.path.exists(artifact_path1) is False assert os.path.exists(artifact_path2) is False diff --git a/pulpcore/tests/unit/models/test_content.py b/pulpcore/tests/unit/models/test_content.py index 319180d64c2..04eb424b8d3 100644 --- a/pulpcore/tests/unit/models/test_content.py +++ b/pulpcore/tests/unit/models/test_content.py @@ -43,7 +43,7 @@ def test_create_read_delete_content(tmp_path): @pytest.mark.django_db def test_storage_location(tmp_path, settings): - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": pytest.skip("Skipping test for nonlocal storage.") tf = tmp_path / "ab" diff --git a/pulpcore/tests/unit/viewsets/test_viewset_base.py b/pulpcore/tests/unit/viewsets/test_viewset_base.py index 7da60311528..da1a0d21a8a 100644 --- a/pulpcore/tests/unit/viewsets/test_viewset_base.py +++ b/pulpcore/tests/unit/viewsets/test_viewset_base.py @@ -1,5 +1,5 @@ import pytest -from pytest_django.asserts import assertQuerysetEqual +from pytest_django.asserts import assertQuerySetEqual import unittest from django.http import Http404, QueryDict @@ -23,7 +23,7 @@ def test_adds_filters(): queryset = viewset.get_queryset() expected = models.RepositoryVersion.objects.filter(repository__pk=repo.pk) - assertQuerysetEqual(queryset, expected) + assertQuerySetEqual(queryset, expected) @pytest.mark.django_db @@ -38,7 +38,7 @@ def test_does_not_add_filters(): queryset = viewset.get_queryset() expected = models.Repository.objects.all() - assertQuerysetEqual(queryset, expected) + assertQuerySetEqual(queryset, expected) def test_must_define_serializer_class(): diff --git a/requirements.txt b/requirements.txt index e5fdec64146..d6580296063 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,10 +6,10 @@ async-timeout>=4.0.3,<4.0.4;python_version<"3.11" backoff>=2.1.2,<2.2.2 click>=8.1.0,<=8.1.7 cryptography>=38.0.1,<42.0.9 -Django~=4.2.0 # LTS version, switch only if we have a compelling reason to +Django>=4.2.24,<5.3, !=5.0, !=5.1 # LTS version, switch only if we have a compelling reason to django-filter>=23.1,<=23.5 django-guid>=3.3,<=3.4.0 -django-import-export>=2.9,<3.4.0 +django-import-export>=2.9,<5.0 django-lifecycle>=1.0,<=1.1.2 djangorestframework>=3.14.0,<=3.15.2 djangorestframework-queryfields>=1.0,<=1.1.0 @@ -37,7 +37,7 @@ pyparsing>=3.1.0,<=3.1.1 python-gnupg>=0.5,<=0.5.2 PyYAML>=5.1.1,<=6.0.1 redis>=4.3,<5.0.3 -tablib<3.6.0 +tablib>=3.5.0,<4.0, !=3.6 url-normalize>=1.4.3,<=1.4.3 uuid6>=2023.5.2,<=2024.1.12 whitenoise>=5.0,<6.7.0 diff --git a/template_config.yml b/template_config.yml index ae01375720f..7342eb12cee 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,8 +1,3 @@ -# This config represents the latest values used when running the plugin-template. Any settings that -# were not present before running plugin-template have been added with their default values. - -# generated with plugin_template - api_root: /pulp/ black: true check_commit_message: true @@ -55,27 +50,23 @@ pulp_settings: - /tmp orphan_protection_time: 0 pulp_settings_azure: - AZURE_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== - AZURE_ACCOUNT_NAME: devstoreaccount1 - AZURE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1; - AZURE_CONTAINER: pulp-test - AZURE_LOCATION: pulp3 - AZURE_OVERWRITE_FILES: true - AZURE_URL_EXPIRATION_SECS: 120 - DEFAULT_FILE_STORAGE: storages.backends.azure_storage.AzureStorage MEDIA_ROOT: '' + STORAGES: + default: + BACKEND: storages.backends.azure_storage.AzureStorage + OPTIONS: + account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + account_name: devstoreaccount1 + connection_string: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1; + azure_container: pulp-test + location: pulp3 + overwrite_files: true + expiration_secs: 120 + staticfiles: + BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage domain_enabled: true pulp_settings_gcp: null pulp_settings_s3: - AWS_ACCESS_KEY_ID: AKIAIT2Z5TDYPX3ARJBA - AWS_DEFAULT_ACL: '@none None' - AWS_S3_ADDRESSING_STYLE: path - AWS_S3_ENDPOINT_URL: http://minio:9000 - AWS_S3_REGION_NAME: eu-central-1 - AWS_S3_SIGNATURE_VERSION: s3v4 - AWS_SECRET_ACCESS_KEY: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS - AWS_STORAGE_BUCKET_NAME: pulp3 - DEFAULT_FILE_STORAGE: storages.backends.s3boto3.S3Boto3Storage MEDIA_ROOT: '' authentication_backends: '@merge django.contrib.auth.backends.RemoteUserBackend' authentication_json_header: HTTP_X_RH_IDENTITY @@ -83,6 +74,20 @@ pulp_settings_s3: domain_enabled: true hide_guarded_distributions: true rest_framework__default_authentication_classes: '@merge pulpcore.app.authentication.JSONHeaderRemoteAuthentication' + STORAGES: + default: + BACKEND: storages.backends.s3boto3.S3Boto3Storage + OPTIONS: + access_key: AKIAIT2Z5TDYPX3ARJBA + secret_key: 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 + bucket_name: pulp3 + endpoint_url: http://minio:9000 + region_name: eu-central-1 + signature_version: s3v4 + addressing_style: path + default_acl: '@none' + staticfiles: + BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage pydocstyle: true release_email: pulp-infra@redhat.com release_user: pulpbot @@ -107,4 +112,3 @@ test_performance: false test_reroute: true test_s3: true use_issue_template: true -