From c613505d53a970af113f5a0bae843377c85105c1 Mon Sep 17 00:00:00 2001 From: Facundo Lopez Janza Date: Tue, 24 Mar 2026 10:28:23 -0300 Subject: [PATCH 1/2] Add BULK_CONSENT_IMPORT_QUEUE_NAME for bulk consent imports Add a dedicated Celery queue (fidesplus.bulk_consent_import) so that bulk pre-verified consent record imports do not compete with the main privacy_preferences queue used by normal consent operations. --- src/fides/api/tasks/__init__.py | 1 + src/fides/api/worker/__init__.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/fides/api/tasks/__init__.py b/src/fides/api/tasks/__init__.py index eabc3fbb380..cf113747a02 100644 --- a/src/fides/api/tasks/__init__.py +++ b/src/fides/api/tasks/__init__.py @@ -30,6 +30,7 @@ DISCOVERY_MONITORS_DETECTION_QUEUE_NAME = "fidesplus.discovery_monitors_detection" # This queue is used for running discovery monitors detection tasks DISCOVERY_MONITORS_CLASSIFICATION_QUEUE_NAME = "fidesplus.discovery_monitors_classification" # This queue is used for running discovery monitors classification tasks DISCOVERY_MONITORS_PROMOTION_QUEUE_NAME = "fidesplus.discovery_monitors_promotion" # This queue is used for running discovery monitors promotion tasks +BULK_CONSENT_IMPORT_QUEUE_NAME = "fidesplus.bulk_consent_import" # This queue is used for bulk importing pre-verified consent records NEW_SESSION_RETRIES = 5 diff --git a/src/fides/api/worker/__init__.py b/src/fides/api/worker/__init__.py index 8ed5e185057..fac7fb2589e 100644 --- a/src/fides/api/worker/__init__.py +++ b/src/fides/api/worker/__init__.py @@ -10,6 +10,7 @@ from fides.api.db.base import Base # type: ignore from fides.api.service.saas_request.override_implementations import * from fides.api.tasks import ( + BULK_CONSENT_IMPORT_QUEUE_NAME, CONSENT_WEBHOOK_QUEUE_NAME, DISCOVERY_MONITORS_CLASSIFICATION_QUEUE_NAME, DISCOVERY_MONITORS_DETECTION_QUEUE_NAME, @@ -80,6 +81,7 @@ def start_worker( DISCOVERY_MONITORS_DETECTION_QUEUE_NAME, DISCOVERY_MONITORS_CLASSIFICATION_QUEUE_NAME, DISCOVERY_MONITORS_PROMOTION_QUEUE_NAME, + BULK_CONSENT_IMPORT_QUEUE_NAME, ] # Fall back to all queues if neither queues nor exclude_queues are provided. From b40614ca77f8260134e148db927002d3237f7787 Mon Sep 17 00:00:00 2001 From: Facundo Lopez Janza Date: Wed, 25 Mar 2026 12:14:43 -0300 Subject: [PATCH 2/2] Address PR review feedback - Add fidesplus.bulk_consent_import to expected queue strings in test_start_worker_with_arguments - Add BULK_CONSENT_IMPORT_QUEUE_NAME to get_queue_counts for health/workers endpoint visibility --- src/fides/api/util/cache.py | 2 ++ tests/ctl/api/test_worker.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/fides/api/util/cache.py b/src/fides/api/util/cache.py index 66d51322c9b..40b25de4282 100644 --- a/src/fides/api/util/cache.py +++ b/src/fides/api/util/cache.py @@ -16,6 +16,7 @@ from fides.api import common_exceptions from fides.api.schemas.masking.masking_secrets import SecretType from fides.api.tasks import ( + BULK_CONSENT_IMPORT_QUEUE_NAME, CONSENT_WEBHOOK_QUEUE_NAME, DISCOVERY_MONITORS_CLASSIFICATION_QUEUE_NAME, DISCOVERY_MONITORS_DETECTION_QUEUE_NAME, @@ -541,6 +542,7 @@ def get_queue_counts() -> Dict[str, int]: DISCOVERY_MONITORS_DETECTION_QUEUE_NAME, DISCOVERY_MONITORS_CLASSIFICATION_QUEUE_NAME, DISCOVERY_MONITORS_PROMOTION_QUEUE_NAME, + BULK_CONSENT_IMPORT_QUEUE_NAME, default_queue_name, ]: queue_counts[queue] = redis_conn.llen(queue) diff --git a/tests/ctl/api/test_worker.py b/tests/ctl/api/test_worker.py index 9080ed8ca0f..1bedfb2ef17 100644 --- a/tests/ctl/api/test_worker.py +++ b/tests/ctl/api/test_worker.py @@ -30,19 +30,19 @@ def test_start_worker_unknown_queue(self, worker_main_mock): ( None, None, - "fides,fidesops.messaging,fides.privacy_preferences,fides.privacy_request_exports,fides.privacy_request_ingestion,fides.dsr,fidesplus.consent_webhooks,fidesplus.discovery_monitors_detection,fidesplus.discovery_monitors_classification,fidesplus.discovery_monitors_promotion", + "fides,fidesops.messaging,fides.privacy_preferences,fides.privacy_request_exports,fides.privacy_request_ingestion,fides.dsr,fidesplus.consent_webhooks,fidesplus.discovery_monitors_detection,fidesplus.discovery_monitors_classification,fidesplus.discovery_monitors_promotion,fidesplus.bulk_consent_import", ), ("fides.dsr", None, "fides.dsr"), ( None, "fides.dsr,fides.privacy_preferences,fides.privacy_request_exports,fides.privacy_request_ingestion,fidesplus.discovery_monitors_detection", - "fides,fidesops.messaging,fidesplus.consent_webhooks,fidesplus.discovery_monitors_classification,fidesplus.discovery_monitors_promotion", + "fides,fidesops.messaging,fidesplus.consent_webhooks,fidesplus.discovery_monitors_classification,fidesplus.discovery_monitors_promotion,fidesplus.bulk_consent_import", ), ("fides,fides.dsr", None, "fides,fides.dsr"), ( None, "fides,fides.dsr", - "fidesops.messaging,fides.privacy_preferences,fides.privacy_request_exports,fides.privacy_request_ingestion,fidesplus.consent_webhooks,fidesplus.discovery_monitors_detection,fidesplus.discovery_monitors_classification,fidesplus.discovery_monitors_promotion", + "fidesops.messaging,fides.privacy_preferences,fides.privacy_request_exports,fides.privacy_request_ingestion,fidesplus.consent_webhooks,fidesplus.discovery_monitors_detection,fidesplus.discovery_monitors_classification,fidesplus.discovery_monitors_promotion,fidesplus.bulk_consent_import", ), ], )