feat: Add repo indexing job (#112136)

Schedule repo indexing job for context engine. This is behind a new
"experimental" feature flag so we can see how this context works
out on sentry seer explorer runs. Only runs index
job on Sunday because we don't want to eat into GH API quotas and
interfere with code review and autofix.

Depends on: getsentry/seer#5594

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: 2 people

src/sentry/features/temporary.py

@@ -302,6 +302,8 @@ def register_temporary_features(manager: FeatureManager) -> None:
     manager.add("organizations:seer-explorer-streaming", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
     # Enable context engine for Seer Explorer
     manager.add("organizations:seer-explorer-context-engine", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
+    # Enable context engine experimental contexts
+    manager.add("organizations:context-engine-experiments", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
     # Enable frontend override for context engine (only for AI/ML/Reasoning platform team)
     manager.add("organizations:seer-explorer-context-engine-allow-fe-override", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
     # Enable frontend override UI component for context engine (only for AI/ML/Reasoning platform team)

src/sentry/seer/autofix/utils.py

@@ -789,6 +789,7 @@ def get_autofix_repos_from_project_code_mappings(
                 "owner": repo_name_sections[0],
                 "name": "/".join(repo_name_sections[1:]),
                 "external_id": repo.external_id,
+                "languages": repo.languages or [],
             }
             repo_key = (repo_dict["provider"], repo_dict["owner"], repo_dict["name"])
 

src/sentry/seer/signed_seer_api.py

@@ -226,6 +226,35 @@ class LlmGenerateRequest(TypedDict):
     response_schema: NotRequired[dict[str, Any]]
 
 
+class RepoDetails(TypedDict):
+    project_ids: list[int]
+    provider: str
+    owner: str
+    name: str
+    external_id: str
+    languages: list[str]
+    integration_id: NotRequired[str | None]
+
+
+class ExplorerIndexOrgRepoRequest(TypedDict):
+    org_id: int
+    repos: list[RepoDetails]
+
+
+def make_org_repo_knowledge_index_request(
+    body: ExplorerIndexOrgRepoRequest,
+    timeout: int | float | None = None,
+    viewer_context: SeerViewerContext | None = None,
+) -> BaseHTTPResponse:
+    return make_signed_seer_api_request(
+        seer_autofix_default_connection_pool,
+        "/v1/automation/explorer/index/org-repo-knowledge",
+        body=orjson.dumps(body),
+        timeout=timeout,
+        viewer_context=viewer_context,
+    )
+
+
 def make_org_project_knowledge_index_request(
     body: OrgProjectKnowledgeIndexRequest,
     timeout: int | float | None = None,

src/sentry/tasks/seer/context_engine_index.py

@@ -14,6 +14,10 @@
 from sentry.models.organization import Organization
 from sentry.models.project import Project
 from sentry.search.events.types import SnubaParams
+from sentry.seer.autofix.utils import (
+    bulk_get_project_preferences,
+    get_autofix_repos_from_project_code_mappings,
+)
 from sentry.seer.explorer.context_engine_utils import (
     EVENT_COUNT_LOOKBACK_DAYS,
     ProjectEventCounts,
@@ -30,12 +34,15 @@
 )
 from sentry.seer.models import SeerApiError
 from sentry.seer.signed_seer_api import (
+    ExplorerIndexOrgRepoRequest,
     ExplorerIndexSentryKnowledgeRequest,
     OrgProjectKnowledgeIndexRequest,
     OrgProjectKnowledgeProjectData,
+    RepoDetails,
     SeerViewerContext,
     make_index_sentry_knowledge_request,
     make_org_project_knowledge_index_request,
+    make_org_repo_knowledge_index_request,
 )
 from sentry.tasks.base import instrumented_task
 from sentry.taskworker.namespaces import seer_tasks
@@ -213,6 +220,90 @@ def build_service_map(organization_id: int, *args, **kwargs) -> None:
         raise
 
 
+@instrumented_task(
+    name="sentry.tasks.seer.context_engine_index.index_repos",
+    namespace=seer_tasks,
+    processing_deadline_duration=10 * 60,  # 10 minutes
+    retry=Retry(times=3, on=(SeerApiError,), delay=60),
+)
+def index_repos(organization_id: int, *args, **kwargs) -> None:
+    if not options.get("explorer.context_engine_indexing.enable"):
+        logger.info("explorer.context_engine_indexing.enable flag is disabled")
+        return
+
+    try:
+        organization = Organization.objects.get(id=organization_id)
+    except Organization.DoesNotExist:
+        logger.error("Organization not found", extra={"org_id": organization_id})
+        return
+
+    if not features.has("organizations:context-engine-experiments", organization):
+        logger.info("organizations:context-engine-experiments flag is disabled")
+        return
+
+    logger.info(
+        "Starting repo index task",
+        extra={"org_id": organization_id},
+    )
+
+    projects = list(
+        Project.objects.filter(organization_id=organization_id, status=ObjectStatus.ACTIVE)
+    )
+    project_map = {p.id: p for p in projects}
+
+    if not project_map:
+        logger.info("No projects found for organization", extra={"org_id": organization_id})
+        return
+
+    org_repo_definitions: dict[tuple[str, str, str], RepoDetails] = {}
+
+    preferences_by_id = bulk_get_project_preferences(organization_id, list(project_map.keys()))
+
+    for project_id, project in project_map.items():
+        existing_pref = preferences_by_id.get(str(project_id))
+        if not existing_pref:
+            continue
+
+        project_pref_repos = existing_pref.get("repositories") or []
+
+        autofix_repos = get_autofix_repos_from_project_code_mappings(project)
+        # Use autofix repos to get repo languages
+        language_map: dict[tuple[str, str, str], list[str]] = {}
+        for autofix_repo in autofix_repos:
+            key = (autofix_repo["provider"], autofix_repo["owner"], autofix_repo["name"])
+            language_map[key] = autofix_repo["languages"]
+
+        for repo in project_pref_repos:
+            key = (repo["provider"], repo["owner"], repo["name"])
+            if key in org_repo_definitions:
+                repo_definition = org_repo_definitions[key]
+                repo_definition["project_ids"].append(project_id)
+            else:
+                org_repo_definitions[key] = {
+                    "project_ids": [project_id],
+                    "provider": repo["provider"],
+                    "owner": repo["owner"],
+                    "name": repo["name"],
+                    "external_id": repo["external_id"],
+                    "languages": language_map.get(key, []),
+                    "integration_id": repo.get("integration_id"),
+                }
+
+    viewer_context = SeerViewerContext(organization_id=organization_id)
+    response = make_org_repo_knowledge_index_request(
+        ExplorerIndexOrgRepoRequest(
+            org_id=organization.id, repos=list(org_repo_definitions.values())
+        ),
+        timeout=30,
+        viewer_context=viewer_context,
+    )
+
+    if response.status >= 400:
+        raise SeerApiError("Seer request failed", response.status)
+
+    logger.info("Successfully indexed repos for org", extra={"org_id": organization_id})
+
+
 def get_allowed_org_ids_context_engine_indexing() -> list[int]:
     """
     Get the list of allowed organizations for context engine indexing.
@@ -283,12 +374,17 @@ def schedule_context_engine_indexing_tasks() -> None:
         return
 
     allowed_org_ids = get_allowed_org_ids_context_engine_indexing()
+    now = datetime.now(UTC)
 
     dispatched = 0
     for org_id in allowed_org_ids:
         try:
             index_org_project_knowledge.apply_async(args=[org_id])
             build_service_map.apply_async(args=[org_id])
+
+            if now.weekday() == 6:  # Sunday
+                index_repos.apply_async(args=[org_id])
+
             dispatched += 1
         except Exception:
             logger.exception(

tests/sentry/autofix/test_utils.py

@@ -44,6 +44,7 @@ def test_get_repos_from_project_code_mappings_with_data(self) -> None:
                 "owner": "getsentry",
                 "name": "sentry",
                 "external_id": "123",
+                "languages": [],
             }
         ]
         assert repos == expected_repos