generalize to work with all integrations

Author: wedamija

src/sentry/conf/server.py

@@ -882,7 +882,6 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "sentry.integrations.github.tasks.codecov_account_unlink",
     "sentry.integrations.github.tasks.link_all_repos",
     "sentry.integrations.github.tasks.pr_comment",
-    "sentry.integrations.github.tasks.sync_repos",
     "sentry.integrations.github.tasks.sync_repos_on_install_change",
     "sentry.integrations.gitlab.tasks",
     "sentry.integrations.jira.tasks",
@@ -892,6 +891,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "sentry.integrations.slack.tasks.link_slack_user_identities",
     "sentry.integrations.slack.tasks.post_message",
     "sentry.integrations.slack.tasks.send_notifications_on_activity",
+    "sentry.integrations.source_code_management.sync_repos",
     "sentry.integrations.source_code_management.tasks",
     "sentry.integrations.tasks.create_comment",
     "sentry.integrations.tasks.kick_off_status_syncs",
@@ -1251,8 +1251,8 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
         "task": "sdk.control:sentry.tasks.release_registry.fetch_release_registry_data_control",
         "schedule": crontab("*/5", "*", "*", "*", "*"),
     },
-    "github-repo-sync-beat": {
-        "task": "integrations.control:sentry.integrations.github.tasks.sync_repos.github_repo_sync_beat",
+    "scm-repo-sync-beat": {
+        "task": "integrations.control:sentry.integrations.source_code_management.sync_repos.scm_repo_sync_beat",
         "schedule": timedelta(minutes=1),
     },
 }

src/sentry/integrations/github/tasks/__init__.py

@@ -2,15 +2,12 @@
 from .codecov_account_unlink import codecov_account_unlink
 from .link_all_repos import link_all_repos
 from .pr_comment import github_comment_workflow
-from .sync_repos import github_repo_sync_beat, sync_repos_for_org
 from .sync_repos_on_install_change import sync_repos_on_install_change
 
 __all__ = (
     "codecov_account_link",
     "codecov_account_unlink",
     "github_comment_workflow",
-    "github_repo_sync_beat",
     "link_all_repos",
-    "sync_repos_for_org",
     "sync_repos_on_install_change",
 )

…/integrations/github/tasks/sync_repos.py …ons/source_code_management/sync_repos.pysrc/sentry/integrations/github/tasks/sync_repos.py renamed to src/sentry/integrations/source_code_management/sync_repos.py src/sentry/integrations/github/tasks/sync_repos.py renamed to src/sentry/integrations/source_code_management/sync_repos.py

@@ -1,19 +1,23 @@
 """
-Periodic repo sync for GitHub integrations.
+Periodic repo sync for SCM integrations.
 
-The beat task (`github_repo_sync_beat`) runs on a schedule and uses
-CursoredScheduler to iterate over all active GitHub OrganizationIntegrations.
-For each one, it dispatches `sync_repos_for_org` which diffs GitHub's repo
-list against Sentry's Repository table and creates/disables/re-enables as needed.
+The beat task (`scm_repo_sync_beat`) runs on a schedule and uses
+CursoredScheduler to iterate over all active SCM OrganizationIntegrations.
+For each one, it dispatches `sync_repos_for_org` which diffs the provider's
+repo list against Sentry's Repository table and creates/disables/re-enables
+as needed.
 """
 
 import logging
+from collections.abc import Mapping
 from datetime import timedelta
+from typing import Any
 
 from taskbroker_client.retry import Retry
 
 from sentry import features
 from sentry.constants import ObjectStatus
+from sentry.features.exceptions import FeatureNotRegistered
 from sentry.integrations.models.organization_integration import OrganizationIntegration
 from sentry.integrations.services.integration import integration_service
 from sentry.integrations.services.repository.service import repository_service
@@ -24,6 +28,7 @@
 from sentry.organizations.services.organization import organization_service
 from sentry.plugins.providers.integration_repository import (
     RepoExistsError,
+    RepositoryInputConfig,
     get_integration_repository_provider,
 )
 from sentry.shared_integrations.exceptions import ApiError
@@ -33,13 +38,37 @@
 from sentry.utils import metrics
 from sentry.utils.cursored_scheduler import CursoredScheduler
 
-from .link_all_repos import get_repo_config
-
 logger = logging.getLogger(__name__)
 
+# All SCM providers that support periodic repo sync
+SCM_PROVIDERS = [
+    "github",
+    "github_enterprise",
+    "gitlab",
+    "bitbucket",
+    "bitbucket_server",
+    "vsts",
+]
+
+
+def _get_repo_config(repo: Mapping[str, Any], integration_id: int) -> RepositoryInputConfig:
+    return {
+        "external_id": str(repo["id"]),
+        "integration_id": integration_id,
+        "identifier": repo["full_name"],
+    }
+
+
+def _has_feature(flag_name: str, org: Any) -> bool:
+    """Check a feature flag, returning False if the flag is not registered."""
+    try:
+        return features.has(flag_name, org)
+    except FeatureNotRegistered:
+        return False
+
 
 @instrumented_task(
-    name="sentry.integrations.github.tasks.sync_repos.sync_repos_for_org",
+    name="sentry.integrations.source_code_management.sync_repos.sync_repos_for_org",
     namespace=integrations_control_tasks,
     retry=Retry(times=3, delay=120),
     processing_deadline_duration=120,
@@ -50,8 +79,8 @@ def sync_repos_for_org(organization_integration_id: int) -> None:
     """
     Sync repositories for a single OrganizationIntegration.
 
-    Fetches all repos from GitHub, diffs against Sentry's Repository table,
-    and creates/disables/re-enables repos as needed.
+    Fetches all repos from the SCM provider, diffs against Sentry's Repository
+    table, and creates/disables/re-enables repos as needed.
     """
     try:
         oi = OrganizationIntegration.objects.get(
@@ -87,23 +116,26 @@ def sync_repos_for_org(organization_integration_id: int) -> None:
         return
 
     rpc_org = org_context.organization
-    if not features.has("organizations:github-repo-auto-sync", rpc_org):
+    provider_key = integration.provider
+
+    # Feature flags are per-provider: organizations:{provider}-repo-auto-sync
+    if not _has_feature(f"organizations:{provider_key}-repo-auto-sync", rpc_org):
         return
 
-    provider = f"integrations:{integration.provider}"
-    dry_run = not features.has("organizations:github-repo-auto-sync-apply", rpc_org)
+    provider = f"integrations:{provider_key}"
+    dry_run = not _has_feature(f"organizations:{provider_key}-repo-auto-sync-apply", rpc_org)
 
     with SCMIntegrationInteractionEvent(
         interaction_type=SCMIntegrationInteractionType.SYNC_REPOS,
         integration_id=integration.id,
         organization_id=organization_id,
-        provider_key=integration.provider,
+        provider_key=provider_key,
     ).capture():
         installation = integration.get_installation(organization_id=organization_id)
         client = installation.get_client()
 
         try:
-            github_repos = client.get_repos()
+            provider_repos = client.get_repos()
         except ApiError as e:
             if installation.is_rate_limited_error(e):
                 logger.info(
@@ -115,7 +147,7 @@ def sync_repos_for_org(organization_integration_id: int) -> None:
                 )
             raise
 
-        github_external_ids = {str(repo["id"]) for repo in github_repos}
+        provider_external_ids = {str(repo["id"]) for repo in provider_repos}
 
         all_repos = repository_service.get_repositories(
             organization_id=organization_id,
@@ -127,22 +159,22 @@ def sync_repos_for_org(organization_integration_id: int) -> None:
             r for r in all_repos if r.status == ObjectStatus.DISABLED and r.external_id
         ]
 
-        sentry_active_ids = {r.external_id for r in active_repos}
-        sentry_disabled_ids = {r.external_id for r in disabled_repos}
+        sentry_active_ids: set[str] = {r.external_id for r in active_repos}  # type: ignore[misc]
+        sentry_disabled_ids: set[str] = {r.external_id for r in disabled_repos}  # type: ignore[misc]
 
-        new_ids = github_external_ids - sentry_active_ids - sentry_disabled_ids
-        removed_ids = sentry_active_ids - github_external_ids
-        restored_ids = sentry_disabled_ids & github_external_ids
+        new_ids = provider_external_ids - sentry_active_ids - sentry_disabled_ids
+        removed_ids = sentry_active_ids - provider_external_ids
+        restored_ids = sentry_disabled_ids & provider_external_ids
 
         metric_tags = {
-            "provider": integration.provider,
+            "provider": provider_key,
             "dry_run": str(dry_run),
         }
         metrics.distribution("scm.repo_sync.new_repos", len(new_ids), tags=metric_tags)
         metrics.distribution("scm.repo_sync.removed_repos", len(removed_ids), tags=metric_tags)
         metrics.distribution("scm.repo_sync.restored_repos", len(restored_ids), tags=metric_tags)
         metrics.distribution(
-            "scm.repo_sync.provider_total", len(github_external_ids), tags=metric_tags
+            "scm.repo_sync.provider_total", len(provider_external_ids), tags=metric_tags
         )
         metrics.distribution(
             "scm.repo_sync.sentry_active", len(sentry_active_ids), tags=metric_tags
@@ -155,11 +187,11 @@ def sync_repos_for_org(organization_integration_id: int) -> None:
             logger.info(
                 "scm.repo_sync.diff",
                 extra={
-                    "provider": integration.provider,
+                    "provider": provider_key,
                     "integration_id": integration.id,
                     "organization_id": organization_id,
                     "dry_run": dry_run,
-                    "provider_total": len(github_external_ids),
+                    "provider_total": len(provider_external_ids),
                     "sentry_active": len(sentry_active_ids),
                     "sentry_disabled": len(sentry_disabled_ids),
                     "new": len(new_ids),
@@ -173,9 +205,9 @@ def sync_repos_for_org(organization_integration_id: int) -> None:
 
         if new_ids:
             integration_repo_provider = get_integration_repository_provider(integration)
-            repo_configs = [
-                get_repo_config(repo, integration.id)
-                for repo in github_repos
+            repo_configs: list[RepositoryInputConfig] = [
+                _get_repo_config(repo, integration.id)
+                for repo in provider_repos
                 if str(repo["id"]) in new_ids
             ]
             if repo_configs:
@@ -204,16 +236,16 @@ def sync_repos_for_org(organization_integration_id: int) -> None:
 
 
 @instrumented_task(
-    name="sentry.integrations.github.tasks.sync_repos.github_repo_sync_beat",
+    name="sentry.integrations.source_code_management.sync_repos.scm_repo_sync_beat",
     namespace=integrations_control_tasks,
     silo_mode=SiloMode.CONTROL,
 )
-def github_repo_sync_beat() -> None:
+def scm_repo_sync_beat() -> None:
     scheduler = CursoredScheduler(
-        name="github_repo_sync",
-        schedule_key="github-repo-sync-beat",
+        name="scm_repo_sync",
+        schedule_key="scm-repo-sync-beat",
         queryset=OrganizationIntegration.objects.filter(
-            integration__provider="github",
+            integration__provider__in=SCM_PROVIDERS,
             integration__status=ObjectStatus.ACTIVE,
             status=ObjectStatus.ACTIVE,
         ),

…grations/github/tasks/test_sync_repos.py …ource_code_management/test_sync_repos.pytests/sentry/integrations/github/tasks/test_sync_repos.py renamed to tests/sentry/integrations/source_code_management/test_sync_repos.py tests/sentry/integrations/github/tasks/test_sync_repos.py renamed to tests/sentry/integrations/source_code_management/test_sync_repos.py

@@ -6,8 +6,8 @@
 
 from sentry.constants import ObjectStatus
 from sentry.integrations.github.integration import GitHubIntegrationProvider
-from sentry.integrations.github.tasks.sync_repos import sync_repos_for_org
 from sentry.integrations.models.organization_integration import OrganizationIntegration
+from sentry.integrations.source_code_management.sync_repos import sync_repos_for_org
 from sentry.models.repository import Repository
 from sentry.silo.base import SiloMode
 from sentry.testutils.cases import IntegrationTestCase
@@ -21,13 +21,13 @@ class SyncReposForOrgTestCase(IntegrationTestCase):
     base_url = "https://api.github.com"
     key = "github"
 
-    def setUp(self):
+    def setUp(self) -> None:
         super().setUp()
         self.oi = OrganizationIntegration.objects.get(
             organization_id=self.organization.id, integration=self.integration
         )
 
-    def _add_repos_response(self, repos):
+    def _add_repos_response(self, repos: list[dict[str, object]]) -> None:
         responses.add(
             responses.GET,
             self.base_url + "/installation/repositories?per_page=100",