feat(github): Handle installation_repositories webhook (#112227)

Re-do of #111864

This failed due to
[SENTRY-5MSP](https://sentry.sentry.io/issues/?project=1&query=SENTRY-5MSP).
This was caused by us not deploying routings to control first, and so we
tried to fire a control task from a cell. That's fixed in
#112226. Other than that, this
is the same as the previous pr and doesn't need re-review

Currently, we only sync the available repositories from Github on
installing the integration. So over time, if new repositories are added
to the github organization, or access to specific repositories is added
or removed, we end up out of sync with which repositories we store in
Sentry.

To fix this, we are going to start handling the
`installation_repositories` webhook. This is fired whenever the
repositories that a github app can access change. This allows us to keep
all the repos in sync.

Note that when access to a repo is removed, we only ever disable the
repo and never delete it. This allows us to keep the history of commits
and so on so far.

Author: wedamija

src/sentry/conf/server.py

@@ -882,6 +882,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "sentry.integrations.github.tasks.codecov_account_unlink",
     "sentry.integrations.github.tasks.link_all_repos",
     "sentry.integrations.github.tasks.pr_comment",
+    "sentry.integrations.github.tasks.sync_repos_on_install_change",
     "sentry.integrations.gitlab.tasks",
     "sentry.integrations.jira.tasks",
     "sentry.integrations.opsgenie.tasks",

src/sentry/features/temporary.py

@@ -137,6 +137,7 @@ def register_temporary_features(manager: FeatureManager) -> None:
     manager.add("organizations:integrations-cursor", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
     manager.add("organizations:integrations-github-copilot-agent", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
     manager.add("organizations:integrations-github-platform-detection", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
+    manager.add("organizations:github-repo-auto-sync", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
     manager.add("organizations:integrations-perforce", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
     manager.add("organizations:integrations-slack-staging", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
     manager.add("organizations:scm-source-context", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)

src/sentry/integrations/bitbucket/repository.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import Any
 
 from sentry.integrations.types import IntegrationProviderSlug
@@ -47,7 +48,7 @@ def get_webhook_secret(self, organization):
         return secret
 
     def build_repository_config(
-        self, organization: RpcOrganization, data: dict[str, Any]
+        self, organization: RpcOrganization, data: Mapping[str, Any]
     ) -> RepositoryConfig:
         installation = self.get_installation(data.get("installation"), organization.id)
         client = installation.get_client()

src/sentry/integrations/bitbucket_server/repository.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from datetime import datetime, timezone
 from typing import Any
 
@@ -35,7 +36,7 @@ def get_repository_data(self, organization, config):
         return config
 
     def build_repository_config(
-        self, organization: RpcOrganization, data: dict[str, Any]
+        self, organization: RpcOrganization, data: Mapping[str, Any]
     ) -> RepositoryConfig:
         installation = self.get_installation(data.get("installation"), organization.id)
         client = installation.get_client()

src/sentry/integrations/github/repository.py

@@ -52,7 +52,7 @@ def get_repository_data(
         return config
 
     def build_repository_config(
-        self, organization: RpcOrganization, data: dict[str, Any]
+        self, organization: RpcOrganization, data: Mapping[str, Any]
     ) -> RepositoryConfig:
         return {
             "name": data["identifier"],

src/sentry/integrations/github/tasks/__init__.py

@@ -2,10 +2,12 @@
 from .codecov_account_unlink import codecov_account_unlink
 from .link_all_repos import link_all_repos
 from .pr_comment import github_comment_workflow
+from .sync_repos_on_install_change import sync_repos_on_install_change
 
 __all__ = (
     "codecov_account_link",
     "codecov_account_unlink",
     "github_comment_workflow",
     "link_all_repos",
+    "sync_repos_on_install_change",
 )

src/sentry/integrations/github/tasks/link_all_repos.py

@@ -1,4 +1,5 @@
 import logging
+from collections.abc import Mapping
 from typing import Any
 
 from taskbroker_client.retry import Retry
@@ -13,6 +14,7 @@
 from sentry.organizations.services.organization import organization_service
 from sentry.plugins.providers.integration_repository import (
     RepoExistsError,
+    RepositoryInputConfig,
     get_integration_repository_provider,
 )
 from sentry.shared_integrations.exceptions import ApiError
@@ -23,9 +25,9 @@
 logger = logging.getLogger(__name__)
 
 
-def get_repo_config(repo, integration_id):
+def get_repo_config(repo: Mapping[str, Any], integration_id: int) -> RepositoryInputConfig:
     return {
-        "external_id": repo["id"],
+        "external_id": str(repo["id"]),
         "integration_id": integration_id,
         "identifier": repo["full_name"],
     }
@@ -77,7 +79,7 @@ def link_all_repos(
 
         integration_repo_provider = get_integration_repository_provider(integration)
 
-        repo_configs: list[dict[str, Any]] = []
+        repo_configs: list[RepositoryInputConfig] = []
         missing_repos = []
         for repo in repositories:
             try:

src/sentry/integrations/github/tasks/sync_repos_on_install_change.py

@@ -0,0 +1,136 @@
+import logging
+from typing import Literal
+
+from taskbroker_client.retry import Retry
+
+from sentry import features
+from sentry.constants import ObjectStatus
+from sentry.integrations.github.webhook_types import GitHubInstallationRepo
+from sentry.integrations.services.integration import integration_service
+from sentry.integrations.services.integration.model import RpcIntegration
+from sentry.integrations.services.repository.service import repository_service
+from sentry.integrations.source_code_management.metrics import (
+    SCMIntegrationInteractionEvent,
+    SCMIntegrationInteractionType,
+)
+from sentry.organizations.services.organization import organization_service
+from sentry.organizations.services.organization.model import RpcOrganization
+from sentry.plugins.providers.integration_repository import (
+    RepoExistsError,
+    RepositoryInputConfig,
+    get_integration_repository_provider,
+)
+from sentry.silo.base import SiloMode
+from sentry.tasks.base import instrumented_task, retry
+from sentry.taskworker.namespaces import integrations_control_tasks
+
+from .link_all_repos import get_repo_config
+
+logger = logging.getLogger(__name__)
+
+
+@instrumented_task(
+    name="sentry.integrations.github.tasks.sync_repos_on_install_change",
+    namespace=integrations_control_tasks,
+    retry=Retry(times=3, delay=120),
+    processing_deadline_duration=120,
+    silo_mode=SiloMode.CONTROL,
+)
+@retry(exclude=(RepoExistsError, KeyError))
+def sync_repos_on_install_change(
+    integration_id: int,
+    action: str,
+    repos_added: list[GitHubInstallationRepo],
+    repos_removed: list[GitHubInstallationRepo],
+    repository_selection: Literal["all", "selected"],
+) -> None:
+    """
+    Handle GitHub installation_repositories webhook events.
+
+    Creates Repository records for newly accessible repos and disables
+    records for repos that are no longer accessible, across all orgs
+    linked to the integration.
+    """
+    result = integration_service.organization_contexts(integration_id=integration_id)
+    integration = result.integration
+    org_integrations = result.organization_integrations
+
+    if integration is None or integration.status != ObjectStatus.ACTIVE:
+        logger.info(
+            "sync_repos_on_install_change.missing_or_inactive_integration",
+            extra={"integration_id": integration_id},
+        )
+        return
+
+    if not org_integrations:
+        logger.info(
+            "sync_repos_on_install_change.no_org_integrations",
+            extra={"integration_id": integration_id},
+        )
+        return
+
+    provider = f"integrations:{integration.provider}"
+
+    for oi in org_integrations:
+        organization_id = oi.organization_id
+        rpc_org = organization_service.get(id=organization_id)
+
+        if rpc_org is None:
+            logger.info(
+                "sync_repos_on_install_change.missing_organization",
+                extra={"organization_id": organization_id},
+            )
+            continue
+
+        if not features.has("organizations:github-repo-auto-sync", rpc_org):
+            continue
+
+        with SCMIntegrationInteractionEvent(
+            interaction_type=SCMIntegrationInteractionType.SYNC_REPOS_ON_INSTALL_CHANGE,
+            integration_id=integration_id,
+            organization_id=organization_id,
+            provider_key=integration.provider,
+        ).capture():
+            _sync_repos_for_org(
+                integration=integration,
+                rpc_org=rpc_org,
+                provider=provider,
+                repos_added=repos_added,
+                repos_removed=repos_removed,
+            )
+
+
+def _sync_repos_for_org(
+    *,
+    integration: RpcIntegration,
+    rpc_org: RpcOrganization,
+    provider: str,
+    repos_added: list[GitHubInstallationRepo],
+    repos_removed: list[GitHubInstallationRepo],
+) -> None:
+    if repos_added:
+        integration_repo_provider = get_integration_repository_provider(integration)
+        repo_configs: list[RepositoryInputConfig] = []
+        for repo in repos_added:
+            try:
+                repo_configs.append(get_repo_config(repo, integration.id))
+            except KeyError:
+                logger.exception("Failed to translate repository config")
+                continue
+
+        if repo_configs:
+            try:
+                integration_repo_provider.create_repositories(
+                    configs=repo_configs, organization=rpc_org
+                )
+            except RepoExistsError:
+                pass
+
+    if repos_removed:
+        external_ids = [str(repo["id"]) for repo in repos_removed]
+        repository_service.disable_repositories_by_external_ids(
+            organization_id=rpc_org.id,
+            integration_id=integration.id,
+            provider=provider,
+            external_ids=external_ids,
+        )

src/sentry/integrations/github/webhook.py

@@ -30,6 +30,7 @@
 from sentry.integrations.github.webhook_types import (
     GITHUB_WEBHOOK_TYPE_HEADER_KEY,
     GithubWebhookType,
+    InstallationRepositoriesEvent,
 )
 from sentry.integrations.pipeline import ensure_integration
 from sentry.integrations.services.integration.model import (
@@ -418,6 +419,57 @@ def _handle_organization_deletion(
             )
 
 
+class InstallationRepositoriesEventWebhook(GitHubWebhook):
+    """
+    Handles installation_repositories events when repos are added to or
+    removed from the GitHub App installation. Runs in control silo.
+
+    https://docs.github.com/en/webhooks/webhook-events-and-payloads#installation_repositories
+    """
+
+    EVENT_TYPE = IntegrationWebhookEventType.INSTALLATION_REPOSITORIES
+
+    def __call__(  # type: ignore[override]
+        self, event: InstallationRepositoriesEvent, host: str | None = None, **kwargs: Any
+    ) -> None:
+        external_id = get_github_external_id(event=event, host=host)
+        if external_id is None:
+            return
+
+        result = integration_service.organization_contexts(
+            provider=self.provider,
+            external_id=external_id,
+        )
+        integration = result.integration
+
+        if integration is None:
+            logger.warning(
+                "github.installation_repositories.missing_integration",
+                extra={"external_id": str(external_id)},
+            )
+            return
+
+        action = event["action"]
+        repos_added = event["repositories_added"]
+        repos_removed = event["repositories_removed"]
+        repository_selection = event["repository_selection"]
+
+        if not repos_added and not repos_removed:
+            return
+
+        from .tasks.sync_repos_on_install_change import sync_repos_on_install_change
+
+        sync_repos_on_install_change.apply_async(
+            kwargs={
+                "integration_id": integration.id,
+                "action": action,
+                "repos_added": repos_added,
+                "repos_removed": repos_removed,
+                "repository_selection": repository_selection,
+            }
+        )
+
+
 class PushEventWebhook(GitHubWebhook):
     """https://developer.github.com/v3/activity/events/types/#pushevent"""
 
@@ -958,6 +1010,7 @@ class GitHubIntegrationsWebhookEndpoint(Endpoint):
     _handlers: dict[GithubWebhookType, type[GitHubWebhook]] = {
         GithubWebhookType.CHECK_RUN: CheckRunEventWebhook,
         GithubWebhookType.INSTALLATION: InstallationEventWebhook,
+        GithubWebhookType.INSTALLATION_REPOSITORIES: InstallationRepositoriesEventWebhook,
         GithubWebhookType.ISSUE: IssuesEventWebhook,
         GithubWebhookType.ISSUE_COMMENT: IssueCommentEventWebhook,
         GithubWebhookType.PULL_REQUEST: PullRequestEventWebhook,

src/sentry/integrations/github/webhook_types.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from enum import StrEnum
+from typing import Any, Literal, TypedDict
 
 GITHUB_WEBHOOK_TYPE_HEADER = "HTTP_X_GITHUB_EVENT"
 GITHUB_WEBHOOK_TYPE_HEADER_KEY = "X-GITHUB-EVENT"
@@ -29,3 +30,18 @@ class GithubWebhookType(StrEnum):
 CELL_PROCESSED_GITHUB_EVENTS = frozenset(
     t.value for t in GithubWebhookType if t not in _CONTROL_ONLY_EVENTS
 )
+
+
+class GitHubInstallationRepo(TypedDict):
+    id: int
+    full_name: str
+    private: bool
+
+
+class InstallationRepositoriesEvent(TypedDict):
+    action: Literal["added", "removed"]
+    installation: dict[str, Any]
+    repositories_added: list[GitHubInstallationRepo]
+    repositories_removed: list[GitHubInstallationRepo]
+    repository_selection: Literal["all", "selected"]
+    sender: dict[str, Any]