From beb5431486ec53b9e956c9078331e806ada00615 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Fri, 6 Feb 2026 18:28:47 -0500 Subject: [PATCH] [PULP-817] Django5 support and adjustments to make imp/exp work with django-imp-exp 4.x. Co-authored-by: David Davis Assisted by: cursor agent Co-authored-by: Cursor --- .github/workflows/scripts/install.sh | 4 +- CHANGES/5324.feature | 1 + CHANGES/6988.feature | 1 + .../functional/api/test_download_policies.py | 9 ++-- pulpcore/app/apps.py | 17 +++++-- pulpcore/app/checks.py | 2 +- pulpcore/app/importexport.py | 6 ++- pulpcore/app/modelresource.py | 5 +- pulpcore/app/serializers/domain.py | 5 +- pulpcore/app/settings.py | 38 +++++++++++--- pulpcore/app/tasks/export.py | 2 +- pulpcore/app/tasks/importer.py | 6 +++ pulpcore/app/util.py | 6 ++- pulpcore/plugin/importexport.py | 3 ++ pulpcore/pytest_plugin.py | 42 +++++++++------- .../api/test_artifact_distribution.py | 6 +-- .../functional/api/test_crd_artifacts.py | 7 ++- .../tests/functional/api/test_crud_domains.py | 20 ++++---- pulpcore/tests/functional/api/test_status.py | 13 +++-- .../api/using_plugin/test_crud_repos.py | 2 +- .../api/using_plugin/test_orphans.py | 22 ++++---- pulpcore/tests/unit/models/test_content.py | 2 +- .../tests/unit/viewsets/test_viewset_base.py | 6 +-- requirements.txt | 6 +-- template_config.yml | 50 ++++++++++--------- 25 files changed, 171 insertions(+), 110 deletions(-) create mode 100644 CHANGES/5324.feature create mode 100644 CHANGES/6988.feature diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index b5f061256f9..6e81e66f9b0 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -105,7 +105,7 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "DISABLED_authentication_backends": "@merge django.contrib.auth.backends.RemoteUserBackend", "DISABLED_authentication_json_header": "HTTP_X_RH_IDENTITY", "DISABLED_authentication_json_header_jq_filter": ".identity.user.username", "DISABLED_authentication_json_header_openapi_security_scheme": {"description": "External OAuth integration", "flows": {"clientCredentials": {"scopes": {"api.console": "grant_access_to_pulp"}, "tokenUrl": "https://your-identity-provider/token/issuer"}}, "type": "oauth2"}, "DISABLED_rest_framework__default_authentication_classes": "@merge pulpcore.app.authentication.JSONHeaderRemoteAuthentication", "MEDIA_ROOT": "", "domain_enabled": true, "hide_guarded_distributions": true}\ +pulp_scenario_settings: {"DISABLED_authentication_backends": "@merge django.contrib.auth.backends.RemoteUserBackend", "DISABLED_authentication_json_header": "HTTP_X_RH_IDENTITY", "DISABLED_authentication_json_header_jq_filter": ".identity.user.username", "DISABLED_authentication_json_header_openapi_security_scheme": {"description": "External OAuth integration", "flows": {"clientCredentials": {"scopes": {"api.console": "grant_access_to_pulp"}, "tokenUrl": "https://your-identity-provider/token/issuer"}}, "type": "oauth2"}, "DISABLED_rest_framework__default_authentication_classes": "@merge pulpcore.app.authentication.JSONHeaderRemoteAuthentication", "MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true, "hide_guarded_distributions": true}\ pulp_scenario_env: {}\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" @@ -119,7 +119,7 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": "", "api_root_rewrite_header": "X-API-Root", "domain_enabled": true}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "api_root_rewrite_header": "X-API-Root", "domain_enabled": true}\ pulp_scenario_env: {}\ ' vars/main.yaml fi diff --git a/CHANGES/5324.feature b/CHANGES/5324.feature new file mode 100644 index 00000000000..d2d6cc140ce --- /dev/null +++ b/CHANGES/5324.feature @@ -0,0 +1 @@ +Adapted PulpImport/Export to allow update django-import-export==4.x. \ No newline at end of file diff --git a/CHANGES/6988.feature b/CHANGES/6988.feature new file mode 100644 index 00000000000..db855921cf8 --- /dev/null +++ b/CHANGES/6988.feature @@ -0,0 +1 @@ +Allow use of Django5 as well as Django4. \ No newline at end of file diff --git a/pulp_file/tests/functional/api/test_download_policies.py b/pulp_file/tests/functional/api/test_download_policies.py index eeaa5a1fb72..338ef7f63c0 100644 --- a/pulp_file/tests/functional/api/test_download_policies.py +++ b/pulp_file/tests/functional/api/test_download_policies.py @@ -11,7 +11,6 @@ from pulpcore.tests.functional.utils import get_files_in_manifest, download_file -from pulpcore.app import settings from pulpcore.client.pulp_file import FileFilePublication, RepositorySyncURL @@ -54,7 +53,7 @@ def test_download_policy( download_policy, ): """Test that "on_demand" and "streamed" download policies work as expected.""" - if download_policy == "on_demand" and "SFTP" in pulp_settings.DEFAULT_FILE_STORAGE: + if download_policy == "on_demand" and "SFTP" in pulp_settings.STORAGES["default"]["BACKEND"]: pytest.skip("This storage technology is not properly supported.") remote = file_remote_ssl_factory( @@ -148,8 +147,8 @@ def test_download_policy( assert expected_checksum == actual_checksum if ( download_policy == "immediate" - and settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem" - and settings.REDIRECT_TO_OBJECT_STORAGE + and pulp_settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem" + and pulp_settings.REDIRECT_TO_OBJECT_STORAGE ): content_disposition = downloaded_file.response_obj.headers.get("Content-Disposition") assert content_disposition is not None @@ -187,7 +186,7 @@ def test_download_policy( content_unit = expected_files_list[4] content_unit_url = urljoin(distribution.base_url, content_unit[0]) # The S3 test API project doesn't handle invalid Range values correctly - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": with pytest.raises(ClientResponseError) as exc: range_header = {"Range": "bytes=-1-11"} download_file(content_unit_url, headers=range_header) diff --git a/pulpcore/app/apps.py b/pulpcore/app/apps.py index f82fedfebe2..e4935f3199b 100644 --- a/pulpcore/app/apps.py +++ b/pulpcore/app/apps.py @@ -5,7 +5,6 @@ from importlib import import_module from django import apps -from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.db import connection, transaction from django.db.models.signals import post_migrate, pre_migrate @@ -68,6 +67,12 @@ class PulpPluginAppConfig(apps.AppConfig): def __init__(self, app_name, app_module): super().__init__(app_name, app_module) + # begin compatibility layer for DEFAULT_FILE_STORAGE deprecation + # Workaround for getting the up-to-date settings instance + from django.conf import settings + + self.settings = settings + # end try: self.version @@ -268,6 +273,8 @@ def _clean_app_status(sender, apps, verbosity, **kwargs): from django.contrib.postgres.functions import TransactionNow from django.utils import timezone + settings = sender.settings + app_ttl_map = [ ("ApiAppStatus", settings.API_APP_TTL), ("ContentAppStatus", settings.CONTENT_APP_TTL), @@ -340,6 +347,7 @@ def _populate_system_id(sender, apps, verbosity, **kwargs): def _ensure_default_domain(sender, **kwargs): + settings = sender.settings table_names = connection.introspection.table_names() if "core_domain" in table_names: from pulpcore.app.util import get_default_domain @@ -349,11 +357,11 @@ def _ensure_default_domain(sender, **kwargs): if ( settings.HIDE_GUARDED_DISTRIBUTIONS != default.hide_guarded_distributions or settings.REDIRECT_TO_OBJECT_STORAGE != default.redirect_to_object_storage - or settings.DEFAULT_FILE_STORAGE != default.storage_class + or settings.STORAGES["default"]["BACKEND"] != default.storage_class ): default.hide_guarded_distributions = settings.HIDE_GUARDED_DISTRIBUTIONS default.redirect_to_object_storage = settings.REDIRECT_TO_OBJECT_STORAGE - default.storage_class = settings.DEFAULT_FILE_STORAGE + default.storage_class = settings.STORAGES["default"]["BACKEND"] default.save(skip_hooks=True) @@ -419,8 +427,9 @@ def _get_permission(perm): def _populate_artifact_serving_distribution(sender, apps, verbosity, **kwargs): + settings = sender.settings if ( - settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem" + settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem" or not settings.REDIRECT_TO_OBJECT_STORAGE ): try: diff --git a/pulpcore/app/checks.py b/pulpcore/app/checks.py index a0129bf56b6..a8d4a26b642 100644 --- a/pulpcore/app/checks.py +++ b/pulpcore/app/checks.py @@ -38,7 +38,7 @@ def secret_key_check(app_configs, **kwargs): def storage_paths(app_configs, **kwargs): warnings = [] - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": try: media_root_dev = Path(settings.MEDIA_ROOT).stat().st_dev except OSError: diff --git a/pulpcore/app/importexport.py b/pulpcore/app/importexport.py index 3dab3aa6ee9..dd30042b134 100644 --- a/pulpcore/app/importexport.py +++ b/pulpcore/app/importexport.py @@ -48,6 +48,10 @@ def _write_export(the_tarfile, resource, dest_dir=None): # the data in batches to memory and concatenate the json lists via string manipulation. with tempfile.NamedTemporaryFile(dir=".", mode="w", encoding="utf8") as temp_file: if isinstance(resource.queryset, QuerySet): + # If we don't have any of "these" - skip writing + if resource.queryset.count() == 0: + return + temp_file.write("[") def process_batch(batch): @@ -114,7 +118,7 @@ def export_artifacts(export, artifact_pks): with ProgressReport(**data) as pb: pb.BATCH_INTERVAL = 5000 - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": with tempfile.TemporaryDirectory(dir=".") as temp_dir: for offset in range(0, len(artifact_pks), EXPORT_BATCH_SIZE): batch = artifact_pks[offset : offset + EXPORT_BATCH_SIZE] diff --git a/pulpcore/app/modelresource.py b/pulpcore/app/modelresource.py index 54d3522fa98..632e982f06a 100644 --- a/pulpcore/app/modelresource.py +++ b/pulpcore/app/modelresource.py @@ -1,3 +1,4 @@ +from django.conf import settings from import_export import fields from import_export.widgets import ForeignKeyWidget from logging import getLogger @@ -36,8 +37,8 @@ def before_import_row(self, row, **kwargs): # the export converts None to blank strings but sha384 and sha512 have unique constraints # that get triggered if they are blank. convert checksums back into None if they are blank. for checksum in ALL_KNOWN_CONTENT_CHECKSUMS: - if row[checksum] == "": - row[checksum] = None + if row[checksum] == "" or checksum not in settings.ALLOWED_CONTENT_CHECKSUMS: + del row[checksum] class Meta: model = Artifact diff --git a/pulpcore/app/serializers/domain.py b/pulpcore/app/serializers/domain.py index f61afd786f7..548fa772237 100644 --- a/pulpcore/app/serializers/domain.py +++ b/pulpcore/app/serializers/domain.py @@ -43,7 +43,10 @@ def to_representation(self, instance): # Should I convert back the saved settings to their Setting names for to_representation? if getattr(self.context.get("domain", None), "name", None) == "default": for setting_name, field in self.SETTING_MAPPING.items(): - if value := getattr(settings, setting_name.upper(), None): + value = getattr(settings, setting_name, None) or settings.STORAGES["default"].get( + "OPTIONS", {} + ).get(field) + if value: instance[field] = value return super().to_representation(instance) diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py index af1766e1757..223d1d25f2b 100644 --- a/pulpcore/app/settings.py +++ b/pulpcore/app/settings.py @@ -16,6 +16,8 @@ from pathlib import Path from cryptography.fernet import Fernet +from django.conf import global_settings +from django.core.files.storage import storages # noqa: F401 from django.core.exceptions import ImproperlyConfigured from django.db import connection @@ -56,7 +58,23 @@ STATIC_URL = "/assets/" STATIC_ROOT = DEPLOY_ROOT / STATIC_URL.strip("/") -DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem" +# begin compatibility layer for DEFAULT_FILE_STORAGE +# Django does not allow STORAGES and DEFAULT_FILE_STORAGE in the same settings module +# (they are mutually exclusive). We set both on global_settings as defaults so that +# users can override either one via dynaconf, and Django picks the right one for its version. +_DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.FileSystem" +_STORAGES = { + "default": { + "BACKEND": "pulpcore.app.models.storage.FileSystem", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, +} +setattr(global_settings, "DEFAULT_FILE_STORAGE", _DEFAULT_FILE_STORAGE) +setattr(global_settings, "STORAGES", _STORAGES) +# end DEFAULT_FILE_STORAGE compatibility layer + REDIRECT_TO_OBJECT_STORAGE = True WORKING_DIRECTORY = DEPLOY_ROOT / "tmp" @@ -363,16 +381,18 @@ from dynaconf import DjangoDynaconf, Validator # noqa # Validators +storage_keys = ("STORAGES.default.BACKEND", "DEFAULT_FILE_STORAGE") storage_validator = ( Validator("REDIRECT_TO_OBJECT_STORAGE", eq=False) - | Validator("DEFAULT_FILE_STORAGE", eq="pulpcore.app.models.storage.FileSystem") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.azure_storage.AzureStorage") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.s3boto3.S3Boto3Storage") - | Validator("DEFAULT_FILE_STORAGE", eq="storages.backends.gcloud.GoogleCloudStorage") + | Validator(*storage_keys, eq="pulpcore.app.models.storage.FileSystem") + | Validator(*storage_keys, eq="storages.backends.azure_storage.AzureStorage") + | Validator(*storage_keys, eq="storages.backends.s3boto3.S3Boto3Storage") + | Validator(*storage_keys, eq="storages.backends.gcloud.GoogleCloudStorage") ) storage_validator.messages["combined"] = ( - "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure" - "storage backend configured in DEFAULT_FILE_STORAGE." + "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure " + "storage backend configured in STORAGES['default']['BACKEND'] " + "(deprecated DEFAULT_FILE_STORAGE)." ) cache_enabled_validator = Validator("CACHE_ENABLED", eq=True) @@ -555,6 +575,10 @@ finally: connection.close() +# Ensures the cached property storage.backends uses the right value after dynaconf init +storages._backends = settings.STORAGES.copy() +storages.backends + if settings.API_ROOT_REWRITE_HEADER: api_root = "//" else: diff --git a/pulpcore/app/tasks/export.py b/pulpcore/app/tasks/export.py index 71956d2359f..e7ef5f164af 100644 --- a/pulpcore/app/tasks/export.py +++ b/pulpcore/app/tasks/export.py @@ -70,7 +70,7 @@ def _export_to_file_system(path, relative_paths_to_artifacts, method=FS_EXPORT_M ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting """ using_filesystem_storage = ( - settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem" + settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem" ) if method != FS_EXPORT_METHODS.WRITE and not using_filesystem_storage: diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py index d26316fbeb2..7eb67fc8059 100644 --- a/pulpcore/app/tasks/importer.py +++ b/pulpcore/app/tasks/importer.py @@ -228,6 +228,9 @@ def _import_file(fpath, resource_class, retry=False): """ try: log.info(f"Importing file {fpath}.") + if not os.path.isfile(fpath): + log.info("...empty - skipping.") + return [] with open(fpath, "r") as json_file: resource = resource_class() log.info(f"...Importing resource {resource.__class__.__name__}.") @@ -236,6 +239,8 @@ def _import_file(fpath, resource_class, retry=False): # overlapping content. for batch_str in _impfile_iterator(json_file): data = Dataset().load(StringIO(batch_str)) + if not data: + return [] if retry: curr_attempt = 1 @@ -267,6 +272,7 @@ def _import_file(fpath, resource_class, retry=False): try: a_result = resource.import_data(data, raise_errors=True) except Exception as e: # noqa log on ANY exception and then re-raise + log.error(e) log.error(f"FATAL import-failure importing {fpath}") raise else: diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py index ec02cbf1d4f..21a05ae60cc 100644 --- a/pulpcore/app/util.py +++ b/pulpcore/app/util.py @@ -528,7 +528,7 @@ def get_artifact_url(artifact, headers=None, http_method=None): if settings.DOMAIN_ENABLED: loc = f"domain {artifact_domain.name}.storage_class" else: - loc = "settings.DEFAULT_FILE_STORAGE" + loc = "settings.STORAGES['default']['BACKEND']" raise NotImplementedError( f"The value {loc}={artifact_domain.storage_class} does not allow redirecting." @@ -574,7 +574,9 @@ def get_default_domain(): try: default_domain = Domain.objects.get(name="default") except Domain.DoesNotExist: - default_domain = Domain(name="default", storage_class=settings.DEFAULT_FILE_STORAGE) + default_domain = Domain( + name="default", storage_class=settings.STORAGES["default"]["BACKEND"] + ) default_domain.save(skip_hooks=True) return default_domain diff --git a/pulpcore/plugin/importexport.py b/pulpcore/plugin/importexport.py index 747a7c0cffa..7ca48e175bc 100644 --- a/pulpcore/plugin/importexport.py +++ b/pulpcore/plugin/importexport.py @@ -41,6 +41,9 @@ def set_up_queryset(self): def dehydrate_pulp_domain(self, content): return str(content.pulp_domain_id) + def render(self, value, obj=None, **kwargs): + return super().render(value, obj, coerce_to_string=False, **kwargs) + def __init__(self, repo_version=None): self.repo_version = repo_version if repo_version: diff --git a/pulpcore/pytest_plugin.py b/pulpcore/pytest_plugin.py index 0c5f706bf77..4dfb49d9c7b 100644 --- a/pulpcore/pytest_plugin.py +++ b/pulpcore/pytest_plugin.py @@ -597,28 +597,36 @@ def _settings_factory(storage_class=None, storage_settings=None): keys = dict() keys["pulpcore.app.models.storage.FileSystem"] = ["MEDIA_ROOT", "MEDIA_URL"] keys["storages.backends.s3boto3.S3Boto3Storage"] = [ - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "AWS_S3_ENDPOINT_URL", - "AWS_S3_ADDRESSING_STYLE", - "AWS_S3_SIGNATURE_VERSION", - "AWS_S3_REGION_NAME", - "AWS_STORAGE_BUCKET_NAME", + "access_key", + "secret_key", + "endpoint_url", + "addressing_style", + "signature_version", + "region_name", + "bucket_name", ] keys["storages.backends.s3.S3Storage"] = keys["storages.backends.s3boto3.S3Boto3Storage"] keys["storages.backends.azure_storage.AzureStorage"] = [ - "AZURE_ACCOUNT_NAME", - "AZURE_CONTAINER", - "AZURE_ACCOUNT_KEY", - "AZURE_URL_EXPIRATION_SECS", - "AZURE_OVERWRITE_FILES", - "AZURE_LOCATION", - "AZURE_CONNECTION_STRING", + "account_name", + "azure_container", + "account_key", + "expiration_secs", + "overwrite_files", + "location", + "connection_string", ] settings = storage_settings or dict() - backend = storage_class or pulp_settings.DEFAULT_FILE_STORAGE - for key in keys[backend]: - if key not in settings: + backend = storage_class or pulp_settings.STORAGES["default"]["BACKEND"] + not_defined_settings = (k for k in keys[backend] if k not in settings) + # Read storage settings from STORAGES.default.OPTIONS when using the default backend + default_backend = pulp_settings.STORAGES["default"]["BACKEND"] + storages_dict = getattr(pulp_settings, "STORAGES", {}) + storage_options = storages_dict.get("default", {}).get("OPTIONS", {}) + if storage_options and backend == default_backend: + for key in not_defined_settings: + settings[key] = storage_options.get(key) + else: + for key in not_defined_settings: settings[key] = getattr(pulp_settings, key, None) return backend, settings diff --git a/pulpcore/tests/functional/api/test_artifact_distribution.py b/pulpcore/tests/functional/api/test_artifact_distribution.py index c743c8a7188..56c7281e85e 100644 --- a/pulpcore/tests/functional/api/test_artifact_distribution.py +++ b/pulpcore/tests/functional/api/test_artifact_distribution.py @@ -2,8 +2,6 @@ import subprocess from hashlib import sha256 -from django.conf import settings - OBJECT_STORAGES = ( "storages.backends.s3boto3.S3Boto3Storage", @@ -13,7 +11,7 @@ ) -def test_artifact_distribution(random_artifact): +def test_artifact_distribution(random_artifact, pulp_settings): artifact_uuid = random_artifact.pulp_href.split("/")[-2] commands = ( @@ -30,7 +28,7 @@ def test_artifact_distribution(random_artifact): hasher = sha256() hasher.update(response.content) assert hasher.hexdigest() == random_artifact.sha256 - if settings.DEFAULT_FILE_STORAGE in OBJECT_STORAGES: + if pulp_settings.STORAGES["default"]["BACKEND"] in OBJECT_STORAGES: content_disposition = response.headers.get("Content-Disposition") assert content_disposition is not None filename = artifact_uuid diff --git a/pulpcore/tests/functional/api/test_crd_artifacts.py b/pulpcore/tests/functional/api/test_crd_artifacts.py index 6a9f13274dc..daa7f176cd3 100644 --- a/pulpcore/tests/functional/api/test_crd_artifacts.py +++ b/pulpcore/tests/functional/api/test_crd_artifacts.py @@ -6,7 +6,6 @@ import uuid import pytest -from django.conf import settings from pulpcore.client.pulpcore import ApiException @@ -145,12 +144,12 @@ def test_upload_mixed_attrs(pulpcore_bindings, pulpcore_random_file): @pytest.mark.parallel -def test_delete_artifact(pulpcore_bindings, pulpcore_random_file, gen_user): +def test_delete_artifact(pulpcore_bindings, pulpcore_random_file, gen_user, pulp_settings): """Verify that the deletion of artifacts is prohibited for both regular users and administrators.""" - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": pytest.skip("this test only works for filesystem storage") - media_root = settings.MEDIA_ROOT + media_root = pulp_settings.MEDIA_ROOT artifact = pulpcore_bindings.ArtifactsApi.create(pulpcore_random_file["name"]) path_to_file = os.path.join(media_root, artifact.file) diff --git a/pulpcore/tests/functional/api/test_crud_domains.py b/pulpcore/tests/functional/api/test_crud_domains.py index e2705d17a3e..0486bd4b3e1 100644 --- a/pulpcore/tests/functional/api/test_crud_domains.py +++ b/pulpcore/tests/functional/api/test_crud_domains.py @@ -5,7 +5,6 @@ import string import json from pulpcore.client.pulpcore import ApiException -from pulpcore.app import settings from pulpcore.tests.functional.utils import PulpTaskError @@ -51,7 +50,7 @@ def test_crud_domains(pulpcore_bindings, monitor_task): @pytest.mark.parallel -def test_default_domain(pulpcore_bindings): +def test_default_domain(pulpcore_bindings, pulp_settings): """Test properties around the default domain.""" domains = pulpcore_bindings.DomainsApi.list(name="default") assert domains.count == 1 @@ -59,9 +58,9 @@ def test_default_domain(pulpcore_bindings): # Read the default domain, ensure storage is set to default default_domain = domains.results[0] assert default_domain.name == "default" - assert default_domain.storage_class == settings.DEFAULT_FILE_STORAGE - assert default_domain.redirect_to_object_storage == settings.REDIRECT_TO_OBJECT_STORAGE - assert default_domain.hide_guarded_distributions == settings.HIDE_GUARDED_DISTRIBUTIONS + assert default_domain.storage_class == pulp_settings.STORAGES["default"]["BACKEND"] + assert default_domain.redirect_to_object_storage == pulp_settings.REDIRECT_TO_OBJECT_STORAGE + assert default_domain.hide_guarded_distributions == pulp_settings.HIDE_GUARDED_DISTRIBUTIONS # Try to create another default domain body = { @@ -92,9 +91,9 @@ def test_default_domain(pulpcore_bindings): @pytest.mark.parallel -def test_active_domain_deletion(pulpcore_bindings, monitor_task): +def test_active_domain_deletion(pulpcore_bindings, monitor_task, pulp_settings): """Test trying to delete a domain that is in use, has objects in it.""" - if not settings.DOMAIN_ENABLED: + if not pulp_settings.DOMAIN_ENABLED: pytest.skip("Domains not enabled") name = str(uuid.uuid4()) body = { @@ -133,9 +132,10 @@ def test_orphan_domain_deletion( gen_object_with_cleanup, monitor_task, tmp_path, + pulp_settings, ): """Test trying to delete a domain that is in use, has objects in it.""" - if not settings.DOMAIN_ENABLED: + if not pulp_settings.DOMAIN_ENABLED: pytest.skip("Domains not enabled") body = { "name": str(uuid.uuid4()), @@ -177,9 +177,9 @@ def test_orphan_domain_deletion( @pytest.mark.parallel -def test_special_domain_creation(pulpcore_bindings, gen_object_with_cleanup): +def test_special_domain_creation(pulpcore_bindings, gen_object_with_cleanup, pulp_settings): """Test many possible domain creation scenarios.""" - if not settings.DOMAIN_ENABLED: + if not pulp_settings.DOMAIN_ENABLED: pytest.skip("Domains not enabled") # This test needs to account for which environment it is running in storage_types = { diff --git a/pulpcore/tests/functional/api/test_status.py b/pulpcore/tests/functional/api/test_status.py index 9f46dd875cf..e3191436d7d 100644 --- a/pulpcore/tests/functional/api/test_status.py +++ b/pulpcore/tests/functional/api/test_status.py @@ -2,7 +2,6 @@ import pytest -from django.conf import settings from jsonschema import validate from pulpcore.client.pulpcore import ApiException @@ -59,24 +58,24 @@ @pytest.mark.parallel -def test_get_authenticated(pulpcore_bindings): +def test_get_authenticated(pulpcore_bindings, pulp_settings): """GET the status path with valid credentials. Verify the response with :meth:`verify_get_response`. """ response = pulpcore_bindings.StatusApi.status_read() - verify_get_response(response.to_dict(), STATUS) + verify_get_response(response.to_dict(), STATUS, pulp_settings) @pytest.mark.parallel -def test_get_unauthenticated(pulpcore_bindings, anonymous_user): +def test_get_unauthenticated(pulpcore_bindings, anonymous_user, pulp_settings): """GET the status path with no credentials. Verify the response with :meth:`verify_get_response`. """ with anonymous_user: response = pulpcore_bindings.StatusApi.status_read() - verify_get_response(response.to_dict(), STATUS) + verify_get_response(response.to_dict(), STATUS, pulp_settings) @pytest.mark.parallel @@ -126,7 +125,7 @@ def test_storage_per_domain( assert default_status.storage != domain_status.storage -def verify_get_response(status, expected_schema): +def verify_get_response(status, expected_schema, pulp_settings): """Verify the response to an HTTP GET call. Verify that several attributes and have the correct type or value. @@ -141,7 +140,7 @@ def verify_get_response(status, expected_schema): assert status["content_settings"]["content_path_prefix"] is not None assert status["storage"]["used"] is not None - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": assert status["storage"]["free"] is None assert status["storage"]["total"] is None else: diff --git a/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py b/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py index 0cb6355a66d..69c6a7b0d02 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py +++ b/pulpcore/tests/functional/api/using_plugin/test_crud_repos.py @@ -329,7 +329,7 @@ def raise_for_invalid_request(remote_attrs): """Check if Pulp returns HTTP 400 after issuing an invalid request.""" with pytest.raises(ApiException) as ae: file_bindings.RemotesFileApi.create(remote_attrs) - assert ae.value.status == 400 + assert ae.value.status == 400 # Test the validation of an invalid absolute pathname. remote_attrs = { diff --git a/pulpcore/tests/functional/api/using_plugin/test_orphans.py b/pulpcore/tests/functional/api/using_plugin/test_orphans.py index 986087cc80b..c6851448635 100644 --- a/pulpcore/tests/functional/api/using_plugin/test_orphans.py +++ b/pulpcore/tests/functional/api/using_plugin/test_orphans.py @@ -3,8 +3,6 @@ import os import pytest -from pulpcore.app import settings - def test_content_orphan_filter( file_bindings, @@ -68,16 +66,17 @@ def test_orphans_delete( random_artifact, file_random_content_unit, monitor_task, + pulp_settings, ): # Verify that the system contains the orphan content unit and the orphan artifact. content_unit = file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href) artifact = pulpcore_bindings.ArtifactsApi.read(random_artifact.pulp_href) - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": # Verify that the artifacts are on disk relative_path = pulpcore_bindings.ArtifactsApi.read(content_unit.artifact).file - artifact_path1 = os.path.join(settings.MEDIA_ROOT, relative_path) - artifact_path2 = os.path.join(settings.MEDIA_ROOT, artifact.file) + artifact_path1 = os.path.join(pulp_settings.MEDIA_ROOT, relative_path) + artifact_path2 = os.path.join(pulp_settings.MEDIA_ROOT, artifact.file) assert os.path.exists(artifact_path1) is True assert os.path.exists(artifact_path2) is True @@ -85,11 +84,11 @@ def test_orphans_delete( monitor_task(pulpcore_bindings.OrphansApi.delete().task) # Assert that the content unit and artifact are gone - if settings.ORPHAN_PROTECTION_TIME == 0: + if pulp_settings.ORPHAN_PROTECTION_TIME == 0: with pytest.raises(file_bindings.ApiException) as exc: file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href) assert exc.value.status == 404 - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": assert os.path.exists(artifact_path1) is False assert os.path.exists(artifact_path2) is False @@ -100,6 +99,7 @@ def test_orphans_cleanup( random_artifact, file_random_content_unit, monitor_task, + pulp_settings, ): # Cleanup orphans with a nonzero orphan_protection_time monitor_task(pulpcore_bindings.OrphansCleanupApi.cleanup({"orphan_protection_time": 10}).task) @@ -108,11 +108,11 @@ def test_orphans_cleanup( content_unit = file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href) artifact = pulpcore_bindings.ArtifactsApi.read(random_artifact.pulp_href) - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": # Verify that the artifacts are on disk relative_path = pulpcore_bindings.ArtifactsApi.read(content_unit.artifact).file - artifact_path1 = os.path.join(settings.MEDIA_ROOT, relative_path) - artifact_path2 = os.path.join(settings.MEDIA_ROOT, artifact.file) + artifact_path1 = os.path.join(pulp_settings.MEDIA_ROOT, relative_path) + artifact_path2 = os.path.join(pulp_settings.MEDIA_ROOT, artifact.file) assert os.path.exists(artifact_path1) is True assert os.path.exists(artifact_path2) is True @@ -123,7 +123,7 @@ def test_orphans_cleanup( with pytest.raises(file_bindings.ApiException) as exc: file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href) assert exc.value.status == 404 - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + if pulp_settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem": assert os.path.exists(artifact_path1) is False assert os.path.exists(artifact_path2) is False diff --git a/pulpcore/tests/unit/models/test_content.py b/pulpcore/tests/unit/models/test_content.py index 319180d64c2..04eb424b8d3 100644 --- a/pulpcore/tests/unit/models/test_content.py +++ b/pulpcore/tests/unit/models/test_content.py @@ -43,7 +43,7 @@ def test_create_read_delete_content(tmp_path): @pytest.mark.django_db def test_storage_location(tmp_path, settings): - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem": pytest.skip("Skipping test for nonlocal storage.") tf = tmp_path / "ab" diff --git a/pulpcore/tests/unit/viewsets/test_viewset_base.py b/pulpcore/tests/unit/viewsets/test_viewset_base.py index 7da60311528..da1a0d21a8a 100644 --- a/pulpcore/tests/unit/viewsets/test_viewset_base.py +++ b/pulpcore/tests/unit/viewsets/test_viewset_base.py @@ -1,5 +1,5 @@ import pytest -from pytest_django.asserts import assertQuerysetEqual +from pytest_django.asserts import assertQuerySetEqual import unittest from django.http import Http404, QueryDict @@ -23,7 +23,7 @@ def test_adds_filters(): queryset = viewset.get_queryset() expected = models.RepositoryVersion.objects.filter(repository__pk=repo.pk) - assertQuerysetEqual(queryset, expected) + assertQuerySetEqual(queryset, expected) @pytest.mark.django_db @@ -38,7 +38,7 @@ def test_does_not_add_filters(): queryset = viewset.get_queryset() expected = models.Repository.objects.all() - assertQuerysetEqual(queryset, expected) + assertQuerySetEqual(queryset, expected) def test_must_define_serializer_class(): diff --git a/requirements.txt b/requirements.txt index b2f866fa6b0..afefa3a5927 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,10 +6,10 @@ async-timeout>=4.0.3,<4.0.4;python_version<"3.11" backoff>=2.1.2,<2.2.2 click>=8.1.0,<=8.1.7 cryptography>=44.0.2,<45.0.4 -Django~=4.2.0 # LTS version, switch only if we have a compelling reason to +Django>=4.2.24,<5.3, !=5.0, !=5.1 # LTS version, switch only if we have a compelling reason to django-filter>=23.1,<=24.3 django-guid>=3.3,<=3.5.0 -django-import-export>=2.9,<3.4.0 +django-import-export>=2.9,<5.0 django-lifecycle>=1.0,<=1.2.4 djangorestframework>=3.14.0,<=3.15.2 djangorestframework-queryfields>=1.0,<=1.1.0 @@ -36,7 +36,7 @@ pyparsing>=3.1.0,<=3.1.4 python-gnupg>=0.5,<=0.5.3 PyYAML>=5.1.1,<=6.0.2 redis>=4.3,<5.0.9 -tablib<3.6.0 +tablib>=3.5.0,<4.0, !=3.6 url-normalize>=1.4.3,<=1.4.3 uuid6>=2023.5.2,<=2024.7.10 whitenoise>=5.0,<6.8.0 diff --git a/template_config.yml b/template_config.yml index 5396762a8ad..ddfac2894a7 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,8 +1,3 @@ -# This config represents the latest values used when running the plugin-template. Any settings that -# were not present before running plugin-template have been added with their default values. - -# generated with plugin_template - api_root: /pulp/ black: true check_commit_message: true @@ -59,28 +54,24 @@ pulp_settings: tmpfile_protection_time: 10 upload_protection_time: 10 pulp_settings_azure: - AZURE_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== - AZURE_ACCOUNT_NAME: devstoreaccount1 - AZURE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1; - AZURE_CONTAINER: pulp-test - AZURE_LOCATION: pulp3 - AZURE_OVERWRITE_FILES: true - AZURE_URL_EXPIRATION_SECS: 120 - DEFAULT_FILE_STORAGE: storages.backends.azure_storage.AzureStorage MEDIA_ROOT: '' + STORAGES: + default: + BACKEND: storages.backends.azure_storage.AzureStorage + OPTIONS: + account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + account_name: devstoreaccount1 + connection_string: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1; + azure_container: pulp-test + location: pulp3 + overwrite_files: true + expiration_secs: 120 + staticfiles: + BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage api_root_rewrite_header: X-API-Root domain_enabled: true pulp_settings_gcp: null pulp_settings_s3: - AWS_ACCESS_KEY_ID: AKIAIT2Z5TDYPX3ARJBA - AWS_DEFAULT_ACL: '@none None' - AWS_S3_ADDRESSING_STYLE: path - AWS_S3_ENDPOINT_URL: http://minio:9000 - AWS_S3_REGION_NAME: eu-central-1 - AWS_S3_SIGNATURE_VERSION: s3v4 - AWS_SECRET_ACCESS_KEY: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS - AWS_STORAGE_BUCKET_NAME: pulp3 - DEFAULT_FILE_STORAGE: storages.backends.s3boto3.S3Boto3Storage DISABLED_authentication_backends: '@merge django.contrib.auth.backends.RemoteUserBackend' DISABLED_authentication_json_header: HTTP_X_RH_IDENTITY DISABLED_authentication_json_header_jq_filter: .identity.user.username @@ -96,6 +87,20 @@ pulp_settings_s3: MEDIA_ROOT: '' domain_enabled: true hide_guarded_distributions: true + STORAGES: + default: + BACKEND: storages.backends.s3boto3.S3Boto3Storage + OPTIONS: + access_key: AKIAIT2Z5TDYPX3ARJBA + secret_key: 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 + bucket_name: pulp3 + endpoint_url: http://minio:9000 + region_name: eu-central-1 + signature_version: s3v4 + addressing_style: path + default_acl: '@none' + staticfiles: + BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage pydocstyle: true release_email: pulp-infra@redhat.com release_user: pulpbot @@ -119,4 +124,3 @@ test_performance: false test_reroute: true test_s3: true use_issue_template: true -