chore(supergroups): Make backfill tuning params configurable via options (#113074)

Move batch_size, inter_batch_delay_s, and max_failures_per_batch from
hardcoded constants to sentry-options so they can be tuned at runtime
without a deploy. We also increased the pace of the backfill as it was
conservative before, defaults reflect new faster backfill values (40,
5s, 20).

Author: yuvmen

src/sentry/options/defaults.py

@@ -1387,6 +1387,24 @@
     default=False,
     flags=FLAG_MODIFIABLE_BOOL | FLAG_AUTOMATOR_MODIFIABLE,
 )
+register(
+    "seer.supergroups_backfill_lightweight.batch_size",
+    type=Int,
+    default=40,
+    flags=FLAG_AUTOMATOR_MODIFIABLE,
+)
+register(
+    "seer.supergroups_backfill_lightweight.inter_batch_delay_s",
+    type=Int,
+    default=5,
+    flags=FLAG_AUTOMATOR_MODIFIABLE,
+)
+register(
+    "seer.supergroups_backfill_lightweight.max_failures_per_batch",
+    type=Int,
+    default=20,
+    flags=FLAG_AUTOMATOR_MODIFIABLE,
+)
 
 # ## sentry.killswitches
 #

src/sentry/tasks/seer/backfill_supergroups_lightweight.py

@@ -25,10 +25,6 @@
 
 logger = logging.getLogger(__name__)
 
-BATCH_SIZE = 25
-INTER_BATCH_DELAY_S = 10
-MAX_FAILURES_PER_BATCH = 10
-
 
 @instrumented_task(
     name="sentry.tasks.seer.backfill_supergroups_lightweight.backfill_supergroups_lightweight_for_org",
@@ -77,6 +73,21 @@ def _backfill_org(
     last_project_id: int,
     last_group_id: int,
 ) -> None:
+    batch_size: int = options.get("seer.supergroups_backfill_lightweight.batch_size")
+    inter_batch_delay_s: int = options.get(
+        "seer.supergroups_backfill_lightweight.inter_batch_delay_s"
+    )
+    max_failures_per_batch: int = options.get(
+        "seer.supergroups_backfill_lightweight.max_failures_per_batch"
+    )
+
+    if batch_size <= 0:
+        logger.error(
+            "supergroups_backfill_lightweight.invalid_batch_size",
+            extra={"organization_id": organization_id, "batch_size": batch_size},
+        )
+        return
+
     # Get the next project to process, starting from where we left off
     project = (
         Project.objects.filter(
@@ -107,7 +118,7 @@ def _backfill_org(
             substatus__in=UNRESOLVED_SUBSTATUS_CHOICES,
         )
         .select_related("project", "project__organization")
-        .order_by("id")[:BATCH_SIZE]
+        .order_by("id")[:batch_size]
     )
 
     if not groups:
@@ -118,7 +129,7 @@ def _backfill_org(
                 "last_project_id": project.id + 1,
                 "last_group_id": 0,
             },
-            countdown=INTER_BATCH_DELAY_S,
+            countdown=inter_batch_delay_s,
             headers={"sentry-propagate-traces": False},
         )
         return
@@ -182,7 +193,7 @@ def _backfill_org(
 
         last_processed_group_id = group.id
 
-        if failure_count >= MAX_FAILURES_PER_BATCH:
+        if max_failures_per_batch > 0 and failure_count >= max_failures_per_batch:
             logger.error(
                 "supergroups_backfill_lightweight.max_failures_reached",
                 extra={
@@ -214,11 +225,11 @@ def _backfill_org(
         },
     )
 
-    if failure_count >= MAX_FAILURES_PER_BATCH:
+    if max_failures_per_batch > 0 and failure_count >= max_failures_per_batch:
         return
 
     # Self-chain: more groups in this project, or move to next project
-    if len(groups) == BATCH_SIZE:
+    if len(groups) == batch_size:
         next_project_id = project.id
         next_group_id = groups[-1].id
     else:
@@ -231,7 +242,7 @@ def _backfill_org(
             "last_project_id": next_project_id,
             "last_group_id": next_group_id,
         },
-        countdown=INTER_BATCH_DELAY_S,
+        countdown=inter_batch_delay_s,
         headers={"sentry-propagate-traces": False},
     )
 

tests/sentry/tasks/seer/test_backfill_supergroups_lightweight.py

@@ -3,10 +3,11 @@
 
 from sentry.models.group import DEFAULT_TYPE_ID
 from sentry.tasks.seer.backfill_supergroups_lightweight import (
-    BATCH_SIZE,
     backfill_supergroups_lightweight_for_org,
 )
 from sentry.testutils.cases import TestCase
+
+TEST_BATCH_SIZE = 5
 from sentry.testutils.helpers.features import with_feature
 from sentry.types.group import GroupSubStatus
 
@@ -81,7 +82,7 @@ def test_self_chains_when_more_groups_exist(self, mock_request):
         mock_request.return_value = MagicMock(status=200)
 
         # Create enough groups to fill a batch
-        for i in range(BATCH_SIZE):
+        for i in range(TEST_BATCH_SIZE):
             evt = self.store_event(
                 data={
                     "message": f"error {i}",
@@ -94,9 +95,12 @@ def test_self_chains_when_more_groups_exist(self, mock_request):
             evt.group.substatus = GroupSubStatus.NEW
             evt.group.save(update_fields=["substatus"])
 
-        with patch(
-            "sentry.tasks.seer.backfill_supergroups_lightweight.backfill_supergroups_lightweight_for_org.apply_async"
-        ) as mock_chain:
+        with (
+            self.options({"seer.supergroups_backfill_lightweight.batch_size": TEST_BATCH_SIZE}),
+            patch(
+                "sentry.tasks.seer.backfill_supergroups_lightweight.backfill_supergroups_lightweight_for_org.apply_async"
+            ) as mock_chain,
+        ):
             backfill_supergroups_lightweight_for_org(self.organization.id)
 
             mock_chain.assert_called_once()
@@ -224,8 +228,8 @@ def test_resumes_from_cursor(self, mock_request):
     def test_chains_then_completes_on_exact_batch_boundary(self, mock_request):
         mock_request.return_value = MagicMock(status=200)
 
-        # Create exactly BATCH_SIZE groups total (setUp already created 1)
-        for i in range(BATCH_SIZE - 1):
+        # Create exactly TEST_BATCH_SIZE groups total (setUp already created 1)
+        for i in range(TEST_BATCH_SIZE - 1):
             evt = self.store_event(
                 data={
                     "message": f"error {i}",
@@ -239,9 +243,12 @@ def test_chains_then_completes_on_exact_batch_boundary(self, mock_request):
             evt.group.save(update_fields=["substatus"])
 
         # First call: full batch, chains with same project and group cursor
-        with patch(
-            "sentry.tasks.seer.backfill_supergroups_lightweight.backfill_supergroups_lightweight_for_org.apply_async"
-        ) as mock_chain:
+        with (
+            self.options({"seer.supergroups_backfill_lightweight.batch_size": TEST_BATCH_SIZE}),
+            patch(
+                "sentry.tasks.seer.backfill_supergroups_lightweight.backfill_supergroups_lightweight_for_org.apply_async"
+            ) as mock_chain,
+        ):
             backfill_supergroups_lightweight_for_org(self.organization.id)
             mock_chain.assert_called_once()
             next_kwargs = mock_chain.call_args.kwargs["kwargs"]
@@ -250,9 +257,12 @@ def test_chains_then_completes_on_exact_batch_boundary(self, mock_request):
 
         # Second call: no groups left in project, chains to next project
         mock_request.reset_mock()
-        with patch(
-            "sentry.tasks.seer.backfill_supergroups_lightweight.backfill_supergroups_lightweight_for_org.apply_async"
-        ) as mock_chain:
+        with (
+            self.options({"seer.supergroups_backfill_lightweight.batch_size": TEST_BATCH_SIZE}),
+            patch(
+                "sentry.tasks.seer.backfill_supergroups_lightweight.backfill_supergroups_lightweight_for_org.apply_async"
+            ) as mock_chain,
+        ):
             backfill_supergroups_lightweight_for_org(self.organization.id, **next_kwargs)
             mock_request.assert_not_called()
             mock_chain.assert_called_once()