chore(integrations): Add canonical way to determine the external id for a repo (#112327)

This is prep work for a pr to genericise the automatic repo syncing pr.
To make it easy to genericise, I needed a way to easily get the external
id for a repo from the client. As part of adding this, I ran into a
bunch of type errors, so this pr also improves typing in this area.

<!-- Describe your PR here. -->

Author: wedamija

src/sentry/integrations/bitbucket/integration.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Mapping, Sequence
+from itertools import chain
 from typing import Any
 
 from django.http.request import HttpRequest
@@ -23,7 +24,10 @@
 from sentry.integrations.models.integration import Integration
 from sentry.integrations.pipeline import IntegrationPipeline
 from sentry.integrations.services.repository import RpcRepository, repository_service
-from sentry.integrations.source_code_management.repository import RepositoryIntegration
+from sentry.integrations.source_code_management.repository import (
+    RepositoryInfo,
+    RepositoryIntegration,
+)
 from sentry.integrations.tasks.migrate_repo import migrate_repo
 from sentry.integrations.types import IntegrationProviderSlug
 from sentry.integrations.utils.atlassian_connect import (
@@ -108,14 +112,14 @@
 scopes = ("issue:write", "pullrequest", "webhook", "repository")
 
 
-class BitbucketIntegration(RepositoryIntegration, BitbucketIssuesSpec):
+class BitbucketIntegration(RepositoryIntegration[BitbucketApiClient], BitbucketIssuesSpec):
     codeowners_locations = [".bitbucket/CODEOWNERS"]
 
     @property
     def integration_name(self) -> str:
         return IntegrationProviderSlug.BITBUCKET.value
 
-    def get_client(self):
+    def get_client(self) -> BitbucketApiClient:
         return BitbucketApiClient(integration=self.model)
 
     # IntegrationInstallation methods
@@ -125,34 +129,50 @@ def error_message_from_json(self, data):
 
     # RepositoryIntegration methods
 
+    def get_repo_external_id(self, repo: Mapping[str, Any]) -> str:
+        return str(repo["uuid"])
+
     def get_repositories(
         self,
         query: str | None = None,
         page_number_limit: int | None = None,
         accessible_only: bool = False,
-    ) -> list[dict[str, Any]]:
+    ) -> list[RepositoryInfo]:
         username = self.model.metadata.get("uuid", self.username)
         if not query:
             resp = self.get_client().get_repos(username)
             return [
-                {"identifier": repo["full_name"], "name": repo["full_name"]}
+                {
+                    "identifier": repo["full_name"],
+                    "name": repo["full_name"],
+                    "external_id": self.get_repo_external_id(repo),
+                }
                 for repo in resp.get("values", [])
             ]
 
+        client = self.get_client()
         exact_query = f'name="{query}"'
         fuzzy_query = f'name~"{query}"'
-        exact_search_resp = self.get_client().search_repositories(username, exact_query)
-        fuzzy_search_resp = self.get_client().search_repositories(username, fuzzy_query)
-
-        result: OrderedSet[str] = OrderedSet()
-
-        for j in exact_search_resp.get("values", []):
-            result.add(j["full_name"])
-
-        for i in fuzzy_search_resp.get("values", []):
-            result.add(i["full_name"])
+        exact_search_resp = client.search_repositories(username, exact_query)
+        fuzzy_search_resp = client.search_repositories(username, fuzzy_query)
+
+        seen: OrderedSet[str] = OrderedSet()
+        repos: list[RepositoryInfo] = []
+        for repo in chain(
+            exact_search_resp.get("values", []),
+            fuzzy_search_resp.get("values", []),
+        ):
+            if repo["full_name"] not in seen:
+                seen.add(repo["full_name"])
+                repos.append(
+                    {
+                        "identifier": repo["full_name"],
+                        "name": repo["full_name"],
+                        "external_id": self.get_repo_external_id(repo),
+                    }
+                )
 
-        return [{"identifier": full_name, "name": full_name} for full_name in result]
+        return repos
 
     def has_repo_access(self, repo: RpcRepository) -> bool:
         client = self.get_client()

src/sentry/integrations/bitbucket/repository.py

@@ -1,5 +1,7 @@
+from __future__ import annotations
+
 from collections.abc import Mapping
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from sentry.integrations.types import IntegrationProviderSlug
 from sentry.locks import locks
@@ -12,8 +14,11 @@
 from sentry.utils.email import parse_email, parse_user_name
 from sentry.utils.http import absolute_uri
 
+if TYPE_CHECKING:
+    from sentry.integrations.bitbucket.integration import BitbucketIntegration  # NOQA
+
 
-class BitbucketRepositoryProvider(IntegrationRepositoryProvider):
+class BitbucketRepositoryProvider(IntegrationRepositoryProvider["BitbucketIntegration"]):
     name = "Bitbucket"
     repo_provider = IntegrationProviderSlug.BITBUCKET.value
 
@@ -25,7 +30,7 @@ def get_repository_data(self, organization, config):
         except Exception as e:
             installation.raise_error(e)
         else:
-            config["external_id"] = str(repo["uuid"])
+            config["external_id"] = installation.get_repo_external_id(repo)
             config["name"] = repo["full_name"]
         return config
 

src/sentry/integrations/bitbucket_server/integration.py

@@ -27,7 +27,10 @@
 from sentry.integrations.pipeline import IntegrationPipeline
 from sentry.integrations.services.repository import repository_service
 from sentry.integrations.services.repository.model import RpcRepository
-from sentry.integrations.source_code_management.repository import RepositoryIntegration
+from sentry.integrations.source_code_management.repository import (
+    RepositoryInfo,
+    RepositoryIntegration,
+)
 from sentry.integrations.tasks.migrate_repo import migrate_repo
 from sentry.integrations.types import IntegrationProviderSlug
 from sentry.integrations.utils.metrics import (
@@ -253,7 +256,7 @@ def dispatch(self, request: HttpRequest, pipeline: IntegrationPipeline) -> HttpR
                 )
 
 
-class BitbucketServerIntegration(RepositoryIntegration):
+class BitbucketServerIntegration(RepositoryIntegration[BitbucketServerClient]):
     """
     IntegrationInstallation implementation for Bitbucket Server
     """
@@ -285,13 +288,14 @@ def get_repositories(
         query: str | None = None,
         page_number_limit: int | None = None,
         accessible_only: bool = False,
-    ) -> list[dict[str, Any]]:
+    ) -> list[RepositoryInfo]:
         if not query:
             resp = self.get_client().get_repos()
 
             return [
                 {
                     "identifier": repo["project"]["key"] + "/" + repo["slug"],
+                    "external_id": self.get_repo_external_id(repo),
                     "project": repo["project"]["key"],
                     "repo": repo["slug"],
                     "name": repo["project"]["name"] + "/" + repo["name"],
@@ -304,6 +308,7 @@ def get_repositories(
         return [
             {
                 "identifier": repo["project"]["key"] + "/" + repo["slug"],
+                "external_id": self.get_repo_external_id(repo),
                 "project": repo["project"]["key"],
                 "repo": repo["slug"],
                 "name": repo["project"]["name"] + "/" + repo["name"],

src/sentry/integrations/bitbucket_server/repository.py

@@ -1,6 +1,8 @@
+from __future__ import annotations
+
 from collections.abc import Mapping
 from datetime import datetime, timezone
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from django.core.cache import cache
 from django.urls import reverse
@@ -15,8 +17,13 @@
 from sentry.utils.hashlib import md5_text
 from sentry.utils.http import absolute_uri
 
+if TYPE_CHECKING:
+    from sentry.integrations.bitbucket_server.integration import BitbucketServerIntegration  # NOQA
+
 
-class BitbucketServerRepositoryProvider(IntegrationRepositoryProvider):
+class BitbucketServerRepositoryProvider(
+    IntegrationRepositoryProvider["BitbucketServerIntegration"]
+):
     name = "Bitbucket Server"
     repo_provider = IntegrationProviderSlug.BITBUCKET_SERVER.value
 
@@ -29,7 +36,7 @@ def get_repository_data(self, organization, config):
         except Exception as e:
             installation.raise_error(e)
         else:
-            config["external_id"] = str(repo["id"])
+            config["external_id"] = installation.get_repo_external_id(repo)
             config["name"] = repo["project"]["key"] + "/" + repo["name"]
             config["project"] = repo["project"]["key"]
             config["repo"] = repo["name"]

src/sentry/integrations/example/integration.py

@@ -22,7 +22,10 @@
 from sentry.integrations.services.integration.serial import serialize_integration
 from sentry.integrations.services.repository.model import RpcRepository
 from sentry.integrations.source_code_management.issues import SourceCodeIssueIntegration
-from sentry.integrations.source_code_management.repository import RepositoryIntegration
+from sentry.integrations.source_code_management.repository import (
+    RepositoryInfo,
+    RepositoryIntegration,
+)
 from sentry.models.repository import Repository
 from sentry.organizations.services.organization.model import RpcOrganization
 from sentry.pipeline.views.base import PipelineView
@@ -151,8 +154,8 @@ def get_repositories(
         query: str | None = None,
         page_number_limit: int | None = None,
         accessible_only: bool = False,
-    ) -> list[dict[str, Any]]:
-        return [{"name": "repo", "identifier": "user/repo"}]
+    ) -> list[RepositoryInfo]:
+        return [{"name": "repo", "identifier": "user/repo", "external_id": "1"}]
 
     def get_unmigratable_repositories(self):
         return []

src/sentry/integrations/github/integration.py

@@ -47,7 +47,10 @@
     PRCommentWorkflow,
 )
 from sentry.integrations.source_code_management.repo_trees import RepoTreesIntegration
-from sentry.integrations.source_code_management.repository import RepositoryIntegration
+from sentry.integrations.source_code_management.repository import (
+    RepositoryInfo,
+    RepositoryIntegration,
+)
 from sentry.integrations.tasks.migrate_repo import migrate_repo
 from sentry.integrations.types import IntegrationProviderSlug
 from sentry.integrations.utils.metrics import (
@@ -237,7 +240,7 @@ def get_document_origin(org) -> str:
 
 
 class GitHubIntegration(
-    RepositoryIntegration,
+    RepositoryIntegration[GitHubBaseClient],
     GitHubIssuesSpec,
     GitHubIssueSyncSpec,
     CommitContextIntegration,
@@ -322,7 +325,7 @@ def get_repositories(
         query: str | None = None,
         page_number_limit: int | None = None,
         accessible_only: bool = False,
-    ) -> list[dict[str, Any]]:
+    ) -> list[RepositoryInfo]:
         """
         args:
         * query - a query to filter the repositories by
@@ -335,30 +338,33 @@ def get_repositories(
         """
         if not query or accessible_only:
             all_repos = self.get_client().get_repos(page_number_limit=page_number_limit)
-            repos = [
+            repos: list[RepositoryInfo] = [
                 {
                     "name": i["name"],
                     "identifier": i["full_name"],
+                    "external_id": self.get_repo_external_id(i),
                     "default_branch": i.get("default_branch"),
                 }
                 for i in all_repos
                 if not i.get("archived")
             ]
             if query:
                 query_lower = query.lower()
-                repos = [r for r in repos if query_lower in r["identifier"].lower()]
+                repos = [r for r in repos if query_lower in str(r["identifier"]).lower()]
             return repos
 
         full_query = build_repository_query(self.model.metadata, self.model.name, query)
         response = self.get_client().search_repositories(full_query)
-        return [
+        search_repos: list[RepositoryInfo] = [
             {
                 "name": i["name"],
                 "identifier": i["full_name"],
+                "external_id": self.get_repo_external_id(i),
                 "default_branch": i.get("default_branch"),
             }
             for i in response.get("items", [])
         ]
+        return search_repos
 
     def get_unmigratable_repositories(self) -> list[RpcRepository]:
         accessible_repos = self.get_repositories()

src/sentry/integrations/github/repository.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Mapping, MutableMapping, Sequence
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from sentry.constants import ObjectStatus
 from sentry.integrations.base import IntegrationInstallation
@@ -15,10 +15,13 @@
 from sentry.plugins.providers.integration_repository import RepositoryConfig
 from sentry.shared_integrations.exceptions import ApiError, IntegrationError
 
+if TYPE_CHECKING:
+    from sentry.integrations.github.integration import GitHubIntegration  # NOQA
+
 WEBHOOK_EVENTS = ["push", "pull_request"]
 
 
-class GitHubRepositoryProvider(IntegrationRepositoryProvider):
+class GitHubRepositoryProvider(IntegrationRepositoryProvider["GitHubIntegration"]):
     name = "GitHub"
     repo_provider = IntegrationProviderSlug.GITHUB.value
 
@@ -46,7 +49,7 @@ def get_repository_data(
         client = installation.get_client()
 
         repo = self._validate_repo(client, installation, config["identifier"])
-        config["external_id"] = str(repo["id"])
+        config["external_id"] = installation.get_repo_external_id(repo)
         config["integration_id"] = installation.model.id
 
         return config

src/sentry/integrations/github_enterprise/integration.py

@@ -37,7 +37,10 @@
 from sentry.integrations.services.integration import integration_service
 from sentry.integrations.services.repository import RpcRepository
 from sentry.integrations.source_code_management.commit_context import CommitContextIntegration
-from sentry.integrations.source_code_management.repository import RepositoryIntegration
+from sentry.integrations.source_code_management.repository import (
+    RepositoryInfo,
+    RepositoryIntegration,
+)
 from sentry.integrations.types import IntegrationProviderSlug
 from sentry.models.repository import Repository
 from sentry.organizations.services.organization import organization_service
@@ -173,7 +176,10 @@ def get_user_info(url, access_token):
 
 
 class GitHubEnterpriseIntegration(
-    RepositoryIntegration, GitHubIssuesSpec, GitHubIssueSyncSpec, CommitContextIntegration
+    RepositoryIntegration[GitHubEnterpriseApiClient],
+    GitHubIssuesSpec,
+    GitHubIssueSyncSpec,
+    CommitContextIntegration,
 ):
     codeowners_locations = ["CODEOWNERS", ".github/CODEOWNERS", "docs/CODEOWNERS"]
 
@@ -220,13 +226,14 @@ def get_repositories(
         query: str | None = None,
         page_number_limit: int | None = None,
         accessible_only: bool = False,
-    ) -> list[dict[str, Any]]:
+    ) -> list[RepositoryInfo]:
         if not query:
             all_repos = self.get_client().get_repos(page_number_limit=page_number_limit)
             return [
                 {
                     "name": i["name"],
                     "identifier": i["full_name"],
+                    "external_id": self.get_repo_external_id(i),
                     "default_branch": i.get("default_branch"),
                 }
                 for i in all_repos
@@ -239,6 +246,7 @@ def get_repositories(
             {
                 "name": i["name"],
                 "identifier": i["full_name"],
+                "external_id": self.get_repo_external_id(i),
                 "default_branch": i.get("default_branch"),
             }
             for i in response.get("items", [])