ref(integrations): Generalize repo sync task to all SCM providers

fixtures/vsts.py

@@ -137,6 +137,11 @@ def _stub_vsts(self):
                         "id": self.repo_id,
                         "name": self.repo_name,
                         "project": {"name": self.project_a["name"]},
+                        "_links": {
+                            "web": {
+                                "href": f"https://{self.vsts_account_name.lower()}.visualstudio.com/_git/{self.repo_name}"
+                            }
+                        },
                     }
                 ]
             },

src/sentry/features/temporary.py

@@ -144,6 +144,14 @@ def register_temporary_features(manager: FeatureManager) -> None:
     manager.add("organizations:github-repo-auto-sync-apply", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
     manager.add("organizations:github_enterprise-repo-auto-sync", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
     manager.add("organizations:github_enterprise-repo-auto-sync-apply", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
+    manager.add("organizations:gitlab-repo-auto-sync", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
+    manager.add("organizations:gitlab-repo-auto-sync-apply", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
+    manager.add("organizations:bitbucket-repo-auto-sync", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
+    manager.add("organizations:bitbucket-repo-auto-sync-apply", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
+    manager.add("organizations:bitbucket_server-repo-auto-sync", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
+    manager.add("organizations:bitbucket_server-repo-auto-sync-apply", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
+    manager.add("organizations:vsts-repo-auto-sync", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
+    manager.add("organizations:vsts-repo-auto-sync-apply", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
     manager.add("organizations:integrations-perforce", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
     manager.add("organizations:integrations-slack-staging", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
     manager.add("organizations:scm-source-context", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)

src/sentry/integrations/github/tasks/link_all_repos.py

@@ -1,6 +1,6 @@
 import logging
 from collections.abc import Mapping
-from typing import Any
+from typing import Any, TypedDict
 
 from taskbroker_client.retry import Retry
 
@@ -12,10 +12,7 @@
     SCMIntegrationInteractionType,
 )
 from sentry.organizations.services.organization import organization_service
-from sentry.plugins.providers.integration_repository import (
-    RepositoryInputConfig,
-    get_integration_repository_provider,
-)
+from sentry.plugins.providers.integration_repository import get_integration_repository_provider
 from sentry.shared_integrations.exceptions import ApiError
 from sentry.silo.base import SiloMode
 from sentry.tasks.base import instrumented_task, retry
@@ -24,7 +21,13 @@
 logger = logging.getLogger(__name__)
 
 
-def get_repo_config(repo: Mapping[str, Any], integration_id: int) -> RepositoryInputConfig:
+class GitHubRepoInputConfig(TypedDict):
+    external_id: str
+    integration_id: int
+    identifier: str
+
+
+def get_repo_config(repo: Mapping[str, Any], integration_id: int) -> GitHubRepoInputConfig:
     return {
         "external_id": str(repo["id"]),
         "integration_id": integration_id,
@@ -78,7 +81,7 @@ def link_all_repos(
 
         integration_repo_provider = get_integration_repository_provider(integration)
 
-        repo_configs: list[RepositoryInputConfig] = []
+        repo_configs: list[GitHubRepoInputConfig] = []
         missing_repos = []
         for repo in repositories:
             try:

src/sentry/integrations/github/tasks/sync_repos_on_install_change.py

@@ -16,15 +16,12 @@
 from sentry.integrations.source_code_management.repo_audit import log_repo_change
 from sentry.organizations.services.organization import organization_service
 from sentry.organizations.services.organization.model import RpcOrganization
-from sentry.plugins.providers.integration_repository import (
-    RepositoryInputConfig,
-    get_integration_repository_provider,
-)
+from sentry.plugins.providers.integration_repository import get_integration_repository_provider
 from sentry.silo.base import SiloMode
 from sentry.tasks.base import instrumented_task, retry
 from sentry.taskworker.namespaces import integrations_control_tasks
 
-from .link_all_repos import get_repo_config
+from .link_all_repos import GitHubRepoInputConfig, get_repo_config
 
 logger = logging.getLogger(__name__)
 
@@ -110,7 +107,7 @@ def _sync_repos_for_org(
 ) -> None:
     if repos_added:
         integration_repo_provider = get_integration_repository_provider(integration)
-        repo_configs: list[RepositoryInputConfig] = []
+        repo_configs: list[GitHubRepoInputConfig] = []
         for repo in repos_added:
             try:
                 repo_configs.append(get_repo_config(repo, integration.id))

src/sentry/integrations/gitlab/integration.py

@@ -184,11 +184,16 @@ def get_repositories(
             # Note: gitlab projects are the same things as repos everywhere else
             group = self.get_group_id()
             resp = self.get_client().search_projects(group, query)
+            instance = self.model.metadata["instance"]
             return [
                 {
                     "identifier": str(repo["id"]),
                     "name": repo["name_with_namespace"],
                     "external_id": self.get_repo_external_id(repo),
+                    "url": repo["web_url"],
+                    "instance": instance,
+                    "path": repo["path_with_namespace"],
+                    "project_id": repo["id"],
                 }
                 for repo in resp
             ]

src/sentry/integrations/source_code_management/repository.py

@@ -36,8 +36,13 @@ class RepositoryInfo(TypedDict):
     identifier: str
     external_id: str
     default_branch: NotRequired[str | None]  # GitHub, GitHub Enterprise
-    project: NotRequired[str]  # Bitbucket Server
+    url: NotRequired[str]  # GitLab, VSTS
+    instance: NotRequired[str]  # GitLab, VSTS
+    project: NotRequired[str]  # Bitbucket Server, VSTS
+    path: NotRequired[str]  # GitLab (path_with_namespace)
+    project_id: NotRequired[int]  # GitLab
     repo: NotRequired[str]  # Bitbucket Server
+    repo_name: NotRequired[str]  # VSTS (bare repo name for API calls)
 
 
 class BaseRepositoryIntegration(ABC):

src/sentry/integrations/source_code_management/sync_repos.py

@@ -28,10 +28,7 @@
 from sentry.integrations.source_code_management.repo_audit import log_repo_change
 from sentry.integrations.source_code_management.repository import RepositoryIntegration
 from sentry.organizations.services.organization import organization_service
-from sentry.plugins.providers.integration_repository import (
-    RepositoryInputConfig,
-    get_integration_repository_provider,
-)
+from sentry.plugins.providers.integration_repository import get_integration_repository_provider
 from sentry.shared_integrations.exceptions import ApiError
 from sentry.silo.base import SiloMode
 from sentry.tasks.base import instrumented_task, retry
@@ -43,16 +40,15 @@
 
 # Providers to include in the periodic sync. Each must implement
 # get_repositories() returning RepositoryInfo with external_id.
+# GitLab, Bitbucket, and Bitbucket Server are excluded because their
+# build_repository_config creates webhooks as a side effect, and
+# create_repositories calls it for every repo before checking if it already
+# exists. This needs to be fixed before adding those providers.
 # Perforce is excluded because it cannot derive external_id from its API.
-# Providers to include in the periodic sync.
-# Other providers (GitLab, Bitbucket, Bitbucket Server, VSTS) have
-# build_repository_config methods that expect additional data beyond what
-# RepositoryInfo provides (e.g. url, instance, project_id). They need
-# follow-up work to either enrich the sync config or simplify their
-# build_repository_config before they can be added here.
 SCM_SYNC_PROVIDERS = [
     "github",
     "github_enterprise",
+    "vsts",
 ]
 
 
@@ -203,11 +199,12 @@ def sync_repos_for_org(organization_integration_id: int) -> None:
 
         if new_ids:
             integration_repo_provider = get_integration_repository_provider(integration)
-            repo_configs: list[RepositoryInputConfig] = [
+            repo_configs = [
                 {
-                    "external_id": repo["external_id"],
-                    "integration_id": integration.id,
+                    **repo,
                     "identifier": str(repo["identifier"]),
+                    "integration_id": integration.id,
+                    "installation": integration.id,
                 }
                 for repo in provider_repos
                 if repo["external_id"] in new_ids

src/sentry/integrations/vsts/integration.py

@@ -325,8 +325,12 @@ def get_repositories(
             data.append(
                 {
                     "name": "{}/{}".format(repo["project"]["name"], repo["name"]),
+                    "repo_name": repo["name"],
                     "identifier": str(repo["id"]),
                     "external_id": self.get_repo_external_id(repo),
+                    "url": repo["_links"]["web"]["href"],
+                    "instance": self.instance,
+                    "project": repo["project"]["name"],
                 }
             )
         return data

src/sentry/integrations/vsts/repository.py

@@ -49,14 +49,15 @@ def get_repository_data(
     def build_repository_config(
         self, organization: RpcOrganization, data: Mapping[str, Any]
     ) -> RepositoryConfig:
+        repo_name = data.get("repo_name", data["name"])
         return {
-            "name": data["name"],
+            "name": repo_name,
             "external_id": data["external_id"],
             "url": data["url"],
             "config": {
                 "instance": data["instance"],
                 "project": data["project"],
-                "name": data["name"],
+                "name": repo_name,
             },
             "integration_id": data["installation"],
         }

src/sentry/plugins/providers/integration_repository.py

@@ -3,7 +3,7 @@
 import logging
 from collections.abc import Mapping
 from datetime import timezone
-from typing import Any, ClassVar, Generic, NotRequired, TypedDict, TypeVar, cast
+from typing import Any, ClassVar, Generic, TypedDict, TypeVar, cast
 
 from dateutil.parser import parse as parse_date
 from rest_framework import status
@@ -30,16 +30,6 @@
 InstT = TypeVar("InstT", bound="RepositoryIntegration[Any]", default=RepositoryIntegration)
 
 
-class RepositoryInputConfig(TypedDict):
-    """Input config passed to create_repositories / build_repository_config.
-    Providers may include additional keys beyond these."""
-
-    external_id: str
-    integration_id: int
-    identifier: str
-    installation: NotRequired[str]
-
-
 class RepositoryConfig(TypedDict):
     name: str
     external_id: str
@@ -240,7 +230,7 @@ def _update_repositories(
 
     def create_repositories(
         self,
-        configs: list[RepositoryInputConfig],
+        configs: list[dict[str, Any]],
         organization: RpcOrganization,
     ) -> tuple[list[RpcRepository], list[RpcRepository], list[RepositoryConfig]]:
         """

tests/sentry/integrations/gitlab/test_issues.py

@@ -60,8 +60,18 @@ def test_get_create_issue_config(self) -> None:
             "https://example.gitlab.com/api/v4/groups/%s/projects"
             % self.installation.model.metadata["group_id"],
             json=[
-                {"name_with_namespace": "getsentry / sentry", "id": 1},
-                {"name_with_namespace": "getsentry / hello", "id": 22},
+                {
+                    "name_with_namespace": "getsentry / sentry",
+                    "id": 1,
+                    "path_with_namespace": "getsentry/sentry",
+                    "web_url": "https://example.gitlab.com/getsentry/sentry",
+                },
+                {
+                    "name_with_namespace": "getsentry / hello",
+                    "id": 22,
+                    "path_with_namespace": "getsentry/hello",
+                    "web_url": "https://example.gitlab.com/getsentry/hello",
+                },
             ],
         )
         assert self.installation.get_create_issue_config(self.group, self.user) == [
@@ -98,8 +108,18 @@ def test_get_link_issue_config(self) -> None:
             "https://example.gitlab.com/api/v4/groups/%s/projects"
             % self.installation.model.metadata["group_id"],
             json=[
-                {"name_with_namespace": "getsentry / sentry", "id": 1},
-                {"name_with_namespace": "getsentry / hello", "id": 22},
+                {
+                    "name_with_namespace": "getsentry / sentry",
+                    "id": 1,
+                    "path_with_namespace": "getsentry/sentry",
+                    "web_url": "https://example.gitlab.com/getsentry/sentry",
+                },
+                {
+                    "name_with_namespace": "getsentry / hello",
+                    "id": 22,
+                    "path_with_namespace": "getsentry/hello",
+                    "web_url": "https://example.gitlab.com/getsentry/hello",
+                },
             ],
         )
         autocomplete_url = "/extensions/gitlab/search/baz/%d/" % self.installation.model.id
@@ -234,9 +254,24 @@ def test_create_issue_default_project_in_group_api_call(self) -> None:
             "https://example.gitlab.com/api/v4/groups/%s/projects"
             % self.installation.model.metadata["group_id"],
             json=[
-                {"name_with_namespace": "getsentry / sentry", "id": 1},
-                {"name_with_namespace": project_name, "id": project_id},
-                {"name_with_namespace": "getsentry / hello", "id": 22},
+                {
+                    "name_with_namespace": "getsentry / sentry",
+                    "id": 1,
+                    "path_with_namespace": "getsentry/sentry",
+                    "web_url": "https://example.gitlab.com/getsentry/sentry",
+                },
+                {
+                    "name_with_namespace": project_name,
+                    "id": project_id,
+                    "path_with_namespace": "this_is/a_project",
+                    "web_url": "https://example.gitlab.com/this_is/a_project",
+                },
+                {
+                    "name_with_namespace": "getsentry / hello",
+                    "id": 22,
+                    "path_with_namespace": "getsentry/hello",
+                    "web_url": "https://example.gitlab.com/getsentry/hello",
+                },
             ],
         )
         responses.add(
@@ -303,14 +338,29 @@ def test_create_issue_default_project_not_in_api_call(self) -> None:
             "https://example.gitlab.com/api/v4/groups/%s/projects"
             % self.installation.model.metadata["group_id"],
             json=[
-                {"name_with_namespace": "getsentry / sentry", "id": 1},
-                {"name_with_namespace": "getsentry / hello", "id": 22},
+                {
+                    "name_with_namespace": "getsentry / sentry",
+                    "id": 1,
+                    "path_with_namespace": "getsentry/sentry",
+                    "web_url": "https://example.gitlab.com/getsentry/sentry",
+                },
+                {
+                    "name_with_namespace": "getsentry / hello",
+                    "id": 22,
+                    "path_with_namespace": "getsentry/hello",
+                    "web_url": "https://example.gitlab.com/getsentry/hello",
+                },
             ],
         )
         responses.add(
             responses.GET,
             "https://example.gitlab.com/api/v4/projects/%s" % project_id,
-            json={"name_with_namespace": project_name, "id": project_id},
+            json={
+                "name_with_namespace": project_name,
+                "id": project_id,
+                "path_with_namespace": "this_is/a_project",
+                "web_url": "https://example.gitlab.com/this_is/a_project",
+            },
         )
         assert self.installation.get_create_issue_config(self.group, self.user) == [
             {