feat(cells): Add GET path for /organization list on control silo (#112622)

- org listing needs to move to the control silo as it returns data relating to orgs that can be spread across multiple cells
- this change introduces the new path, while keeping the existing cell-based org listing in place
- the new implementation uses the control silo variants `OrganizationMapping` and `OrganizationMemberMapping` in place of Organization and `OrganizationMember`
- `owner=1` is not supported currently. This is used for account closing and can be moved to a separate (control-based) api
- platform filtering exists on the cell but is not ported to control - this appears to be older, unused functionality and is not worth porting since this information is not be available in control
- added a TODO for avatar functionality: this should also be available in the control endpoint, but we need to sync avatars to control as a prerequisite. This will be tackled separately.
- member sorting functionality (used in _admin) had to be implemented a bit differently since there are no FK relations but works the same

Author: lynnagara

src/sentry/api/serializers/models/organization.py

@@ -78,6 +78,7 @@
 from sentry.models.options.project_option import ProjectOption
 from sentry.models.organization import Organization, OrganizationStatus
 from sentry.models.organizationaccessrequest import OrganizationAccessRequest
+from sentry.models.organizationmapping import OrganizationMapping
 from sentry.models.organizationonboardingtask import OrganizationOnboardingTask
 from sentry.models.project import Project
 from sentry.models.team import Team, TeamStatus
@@ -285,6 +286,23 @@ def serialize(
         )
 
 
+class ControlSiloOrganizationMappingSerializer(Serializer):
+    # TODO(cells): Add the `avatar` to this serializer
+    # once it is available in the control silo
+    def serialize(
+        self,
+        obj: OrganizationMapping,
+        attrs: Mapping[str, Any],
+        user: User | RpcUser | AnonymousUser,
+        **kwargs: Any,
+    ) -> ControlSiloOrganizationSerializerResponse:
+        return dict(
+            id=str(obj.organization_id),
+            slug=obj.slug,
+            name=obj.name,
+        )
+
+
 @register(Organization)
 class OrganizationSummarySerializer(Serializer):
     def get_attrs(

src/sentry/core/endpoints/organization_index.py

@@ -3,7 +3,8 @@
 import sentry_sdk
 from django.conf import settings
 from django.db import IntegrityError
-from django.db.models import Count, Q
+from django.db.models import Count, OuterRef, Q, Subquery
+from django.db.models.functions import Coalesce
 from drf_spectacular.utils import extend_schema
 from rest_framework import serializers, status
 from rest_framework.request import Request
@@ -16,12 +17,13 @@
 )
 from sentry.analytics.events.organization_created import OrganizationCreatedEvent
 from sentry.api.api_publish_status import ApiPublishStatus
-from sentry.api.base import Endpoint, cell_silo_endpoint
+from sentry.api.base import Endpoint, all_silo_endpoint
 from sentry.api.bases.organization import OrganizationPermission
 from sentry.api.paginator import DateTimePaginator, OffsetPaginator
 from sentry.api.serializers import serialize
 from sentry.api.serializers.models.organization import (
     BaseOrganizationSerializer,
+    ControlSiloOrganizationMappingSerializer,
     OrganizationSummarySerializerResponse,
 )
 from sentry.apidocs.constants import RESPONSE_FORBIDDEN, RESPONSE_NOT_FOUND, RESPONSE_UNAUTHORIZED
@@ -33,7 +35,9 @@
 from sentry.demo_mode.utils import is_demo_user
 from sentry.hybridcloud.rpc import IDEMPOTENCY_KEY_LENGTH
 from sentry.models.organization import Organization, OrganizationStatus
+from sentry.models.organizationmapping import OrganizationMapping
 from sentry.models.organizationmember import OrganizationMember
+from sentry.models.organizationmembermapping import OrganizationMemberMapping
 from sentry.models.projectplatform import ProjectPlatform
 from sentry.search.utils import tokenize_query
 from sentry.services.organization import (
@@ -43,6 +47,7 @@
 )
 from sentry.services.organization.provisioning import organization_provisioning_service
 from sentry.signals import org_setup_complete, terms_accepted
+from sentry.silo.base import SiloMode
 from sentry.users.services.user.service import user_service
 from sentry.utils.pagination_factory import PaginatorLike
 
@@ -69,7 +74,7 @@ def validate_agreeTerms(self, value):
 
 
 @extend_schema(tags=["Users"])
-@cell_silo_endpoint
+@all_silo_endpoint
 class OrganizationIndexEndpoint(Endpoint):
     publish_status = {
         "GET": ApiPublishStatus.PUBLIC,
@@ -101,6 +106,12 @@ def get(self, request: Request) -> Response:
         Return a list of organizations available to the authenticated session in a region.
         This is particularly useful for requests with a user bound context. For API key-based requests this will only return the organization that belongs to the key.
         """
+        if SiloMode.get_current_mode() == SiloMode.CONTROL:
+            return self._get_from_control(request)
+
+        return self._get_from_cell(request)
+
+    def _get_from_cell(self, request: Request) -> Response:
         owner_only = request.GET.get("owner") in ("1", "true")
 
         queryset = Organization.objects.distinct()
@@ -163,6 +174,10 @@ def get(self, request: Request) -> Response:
                     }
                     queryset = queryset.filter(Q(member_set__user_id__in=user_ids))
                 elif key == "platform":
+                    # Note: platform filtering is kept here but is not present in the control version
+                    # of this endpoint, since the data is not in control and our UI isn't
+                    # passing this anymore.
+                    sentry_sdk.capture_message("organization_index.platform_filter_used")
                     queryset = queryset.filter(
                         project__in=ProjectPlatform.objects.filter(platform__in=value).values(
                             "project_id"
@@ -193,6 +208,7 @@ def get(self, request: Request) -> Response:
             order_by = "-member_count"
             paginator_cls = OffsetPaginator
         elif sort_by == "projects":
+            sentry_sdk.capture_message("organization_index.sort_by_projects_used")
             queryset = queryset.annotate(project_count=Count("project"))
             order_by = "-project_count"
             paginator_cls = OffsetPaginator
@@ -208,6 +224,119 @@ def get(self, request: Request) -> Response:
             paginator_cls=paginator_cls,
         )
 
+    def _get_from_control(self, request: Request) -> Response:
+        owner_only = request.GET.get("owner") in ("1", "true")
+
+        if owner_only:
+            return Response(
+                {"detail": "The control-silo organizations endpoint does not support owner=1."},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        queryset = OrganizationMapping.objects.distinct()
+
+        if request.auth and not request.user.is_authenticated:
+            if hasattr(request.auth, "project"):
+                queryset = queryset.filter(organization_id=request.auth.project.organization_id)
+            elif request.auth.organization_id is not None:
+                queryset = queryset.filter(organization_id=request.auth.organization_id)
+        elif not (is_active_superuser(request) and request.GET.get("show") == "all"):
+            assert request.user.id is not None
+            queryset = queryset.filter(
+                organization_id__in=OrganizationMemberMapping.objects.filter(
+                    user_id=request.user.id
+                ).values("organization_id")
+            )
+            if request.auth and request.auth.organization_id is not None and queryset.count() > 1:
+                # If a token is limited to one organization, this endpoint should only return that one organization
+                queryset = queryset.filter(organization_id=request.auth.organization_id)
+
+        query = request.GET.get("query")
+        if query:
+            tokens = tokenize_query(query)
+            for key, value in tokens.items():
+                if key == "query":
+                    query_value = " ".join(value)
+                    user_ids = {
+                        u.id
+                        for u in user_service.get_many_by_email(
+                            emails=[query_value], is_verified=False
+                        )
+                    }
+                    queryset = queryset.filter(
+                        Q(name__icontains=query_value)
+                        | Q(slug__icontains=query_value)
+                        # Control-side equivalent of `Q(member_set__user_id__in=user_ids)`
+                        | Q(
+                            organization_id__in=OrganizationMemberMapping.objects.filter(
+                                user_id__in=user_ids
+                            ).values("organization_id")
+                        )
+                    )
+                elif key == "slug":
+                    queryset = queryset.filter(in_iexact("slug", value))
+                elif key == "email":
+                    user_ids = {
+                        u.id
+                        for u in user_service.get_many_by_email(emails=value, is_verified=False)
+                    }
+                    queryset = queryset.filter(
+                        organization_id__in=OrganizationMemberMapping.objects.filter(
+                            user_id__in=user_ids
+                        ).values("organization_id")
+                    )
+                elif key == "id":
+                    queryset = queryset.filter(organization_id__in=value)
+                elif key == "status":
+                    try:
+                        queryset = queryset.filter(
+                            status__in=[OrganizationStatus[v.upper()] for v in value]
+                        )
+                    except KeyError:
+                        queryset = queryset.none()
+                elif key == "member_id":
+                    queryset = queryset.filter(
+                        organization_id__in=OrganizationMemberMapping.objects.filter(
+                            organizationmember_id__in=value
+                        ).values("organization_id")
+                    )
+                else:
+                    queryset = queryset.none()
+
+        sort_by = request.GET.get("sortBy")
+        paginator_cls: type[PaginatorLike]
+        if sort_by == "members":
+            member_count_subquery = (
+                OrganizationMemberMapping.objects.filter(
+                    organization_id=OuterRef("organization_id")
+                )
+                .values("organization_id")
+                .annotate(member_count=Count("id"))
+                .values("member_count")[:1]
+            )
+            queryset = queryset.annotate(member_count=Coalesce(Subquery(member_count_subquery), 0))
+            order_by = "-member_count"
+            paginator_cls = OffsetPaginator
+        elif sort_by == "projects":
+            queryset = queryset.none()
+            order_by = "-date_created"
+            paginator_cls = OffsetPaginator
+        else:
+            order_by = "-date_created"
+            paginator_cls = DateTimePaginator
+
+        return self.paginate(
+            request=request,
+            queryset=queryset,
+            order_by=order_by,
+            on_results=lambda x: serialize(
+                x,
+                request.user,
+                serializer=ControlSiloOrganizationMappingSerializer(),
+            ),
+            paginator_cls=paginator_cls,
+        )
+
     # XXX: endpoint useless for end-users as it needs user context.
     def post(self, request: Request) -> Response:
         """
@@ -225,6 +354,13 @@ def post(self, request: Request) -> Response:
                                 terms of service and privacy policy.
         :auth: required, user-context-needed
         """
+        # TODO(cells): Move org creation to control as part of the broader
+        # org-listing/org-provisioning cutover. Since POST is private, the
+        # legacy cell-side path can be removed once the control implementation
+        # is ready.
+        if SiloMode.get_current_mode() == SiloMode.CONTROL:
+            return Response(status=404)
+
         if not request.user.is_authenticated:
             return Response({"detail": "This endpoint requires user info"}, status=401)
 

tests/sentry/core/endpoints/test_organization_index.py

@@ -136,6 +136,61 @@ def test_show_only_token_organization(self) -> None:
         assert response.data[0]["id"] == str(org1.id)
 
 
+@control_silo_test(cells=create_test_cells("us", "de"))
+class OrganizationsControlListTest(OrganizationIndexTest):
+    endpoint = "sentry-api-0-organizations"
+
+    def test_membership_across_cells(self) -> None:
+        us_org = self.create_organization(cell="us", owner=self.user, name="US Org", slug="us-org")
+        de_org = self.create_organization(cell="de", owner=self.user, name="DE Org", slug="de-org")
+
+        response = self.get_success_response()
+
+        assert {item["id"] for item in response.data} == {str(us_org.id), str(de_org.id)}
+        assert {item["slug"] for item in response.data} == {"us-org", "de-org"}
+
+    def test_show_only_token_organization(self) -> None:
+        org1 = self.create_organization(cell="us", owner=self.user)
+        self.create_organization(cell="de", owner=self.user)
+
+        with assume_test_silo_mode(SiloMode.CONTROL):
+            org_scoped_token = ApiToken.objects.create(
+                user=self.user, scoping_organization_id=org1.id, scope_list=["org:read"]
+            )
+
+        self.client.credentials(HTTP_AUTHORIZATION=f"Bearer {org_scoped_token.plaintext_token}")
+        response = self.client.get(reverse(self.endpoint))
+
+        assert len(response.data) == 1
+        assert response.data[0]["id"] == str(org1.id)
+
+    def test_owner_not_supported(self) -> None:
+        self.create_organization(cell="us", owner=self.user)
+
+        response = self.get_error_response(status_code=400, owner="1")
+
+        assert (
+            response.data["detail"]
+            == "The control-silo organizations endpoint does not support owner=1."
+        )
+
+    def test_sort_by_members(self) -> None:
+        smaller_org = self.create_organization(
+            cell="us", owner=self.user, name="Smaller Org", slug="smaller-org"
+        )
+        larger_org = self.create_organization(
+            cell="de", owner=self.user, name="Larger Org", slug="larger-org"
+        )
+
+        self.create_member(user=self.create_user(), organization=smaller_org)
+        self.create_member(user=self.create_user(), organization=larger_org)
+        self.create_member(user=self.create_user(), organization=larger_org)
+
+        response = self.get_success_response(sortBy="members")
+
+        assert [item["id"] for item in response.data] == [str(larger_org.id), str(smaller_org.id)]
+
+
 class OrganizationsCreateTest(OrganizationIndexTest, HybridCloudTestMixin):
     method = "post"