feat(autofix): Add analytics events for autofix phase start and completion

src/sentry/analytics/events/__init__.py

@@ -8,6 +8,7 @@
 from .api_token_deleted import *  # noqa: F401,F403
 from .auth_v2 import *  # noqa: F401,F403
 from .autofix_automation_events import *  # noqa: F401,F403
+from .autofix_events import *  # noqa: F401,F403
 from .checkin_processing_error_stored import *  # noqa: F401,F403
 from .codeowners_assignment import *  # noqa: F401,F403
 from .codeowners_created import *  # noqa: F401,F403

src/sentry/analytics/events/autofix_events.py

@@ -0,0 +1,89 @@
+from sentry import analytics
+
+
+@analytics.eventclass()
+class AiAutofixPhaseEvent(analytics.Event):
+    organization_id: int
+    project_id: int
+    group_id: int
+    referrer: str | None
+
+
+@analytics.eventclass("ai.autofix.root_cause.started")
+class AiAutofixRootCauseStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.solution.started")
+class AiAutofixSolutionStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.code_changes.started")
+class AiAutofixCodeChangesStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.root_cause.completed")
+class AiAutofixRootCauseCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.solution.completed")
+class AiAutofixSolutionCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.code_changes.completed")
+class AiAutofixCodeChangesCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.impact_assessment.started")
+class AiAutofixImpactAssessmentStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.impact_assessment.completed")
+class AiAutofixImpactAssessmentCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.triage.started")
+class AiAutofixTriageStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.triage.completed")
+class AiAutofixTriageCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.pr_created.started")
+class AiAutofixPrCreatedStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.pr_created.completed")
+class AiAutofixPrCreatedCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.agent_handoff")
+class AiAutofixAgentHandoffEvent(AiAutofixPhaseEvent):
+    pass
+
+
+analytics.register(AiAutofixRootCauseStartedEvent)
+analytics.register(AiAutofixSolutionStartedEvent)
+analytics.register(AiAutofixCodeChangesStartedEvent)
+analytics.register(AiAutofixRootCauseCompletedEvent)
+analytics.register(AiAutofixSolutionCompletedEvent)
+analytics.register(AiAutofixCodeChangesCompletedEvent)
+analytics.register(AiAutofixImpactAssessmentStartedEvent)
+analytics.register(AiAutofixImpactAssessmentCompletedEvent)
+analytics.register(AiAutofixTriageStartedEvent)
+analytics.register(AiAutofixTriageCompletedEvent)
+analytics.register(AiAutofixPrCreatedStartedEvent)
+analytics.register(AiAutofixPrCreatedCompletedEvent)
+analytics.register(AiAutofixAgentHandoffEvent)

src/sentry/seer/autofix/autofix_agent.py

@@ -8,6 +8,22 @@
 from pydantic import BaseModel
 from rest_framework.exceptions import PermissionDenied
 
+from sentry import analytics
+from sentry.analytics.events.autofix_events import (
+    AiAutofixAgentHandoffEvent,
+    AiAutofixCodeChangesCompletedEvent,
+    AiAutofixCodeChangesStartedEvent,
+    AiAutofixImpactAssessmentCompletedEvent,
+    AiAutofixImpactAssessmentStartedEvent,
+    AiAutofixPhaseEvent,
+    AiAutofixPrCreatedStartedEvent,
+    AiAutofixRootCauseCompletedEvent,
+    AiAutofixRootCauseStartedEvent,
+    AiAutofixSolutionCompletedEvent,
+    AiAutofixSolutionStartedEvent,
+    AiAutofixTriageCompletedEvent,
+    AiAutofixTriageStartedEvent,
+)
 from sentry.constants import ENABLE_SEER_CODING_DEFAULT
 from sentry.seer.autofix.artifact_schemas import (
     ImpactAssessmentArtifact,
@@ -82,34 +98,48 @@ def __init__(
         artifact_schema: type[BaseModel] | None,
         prompt_fn: Callable[..., str],
         enable_coding: bool = False,
+        started_event: type[AiAutofixPhaseEvent] | None = None,
+        completed_event: type[AiAutofixPhaseEvent] | None = None,
     ):
         self.artifact_schema = artifact_schema
         self.prompt_fn = prompt_fn
         self.enable_coding = enable_coding
+        self.started_event = started_event
+        self.completed_event = completed_event
 
 
 # Step configurations mapping step to its artifact schema and prompt
 STEP_CONFIGS: dict[AutofixStep, StepConfig] = {
     AutofixStep.ROOT_CAUSE: StepConfig(
         artifact_schema=RootCauseArtifact,
         prompt_fn=root_cause_prompt,
+        started_event=AiAutofixRootCauseStartedEvent,
+        completed_event=AiAutofixRootCauseCompletedEvent,
     ),
     AutofixStep.SOLUTION: StepConfig(
         artifact_schema=SolutionArtifact,
         prompt_fn=solution_prompt,
+        started_event=AiAutofixSolutionStartedEvent,
+        completed_event=AiAutofixSolutionCompletedEvent,
     ),
     AutofixStep.CODE_CHANGES: StepConfig(
         artifact_schema=None,  # Code changes read from file_patches
         prompt_fn=code_changes_prompt,
         enable_coding=True,
+        started_event=AiAutofixCodeChangesStartedEvent,
+        completed_event=AiAutofixCodeChangesCompletedEvent,
     ),
     AutofixStep.IMPACT_ASSESSMENT: StepConfig(
         artifact_schema=ImpactAssessmentArtifact,
         prompt_fn=impact_assessment_prompt,
+        started_event=AiAutofixImpactAssessmentStartedEvent,
+        completed_event=AiAutofixImpactAssessmentCompletedEvent,
     ),
     AutofixStep.TRIAGE: StepConfig(
         artifact_schema=TriageArtifact,
         prompt_fn=triage_prompt,
+        started_event=AiAutofixTriageStartedEvent,
+        completed_event=AiAutofixTriageCompletedEvent,
     ),
 }
 
@@ -215,6 +245,16 @@ def trigger_autofix_explorer(
     """
 
     config = STEP_CONFIGS[step]
+
+    if config.started_event is not None:
+        analytics.record(
+            config.started_event(
+                organization_id=group.organization.id,
+                project_id=group.project_id,
+                group_id=group.id,
+                referrer=referrer.value,
+            )
+        )
     client = get_autofix_explorer_client(
         group,
         intelligence_level=intelligence_level,
@@ -500,6 +540,15 @@ def trigger_coding_agent_handoff(
         auto_create_pr=auto_create_pr,
     )
 
+    analytics.record(
+        AiAutofixAgentHandoffEvent(
+            organization_id=group.organization.id,
+            project_id=group.project_id,
+            group_id=group.id,
+            referrer=referrer.value,
+        )
+    )
+
     metrics.incr(
         "autofix.explorer.trigger",
         tags={"step": "coding_agent_handoff", "referrer": referrer.value},
@@ -532,6 +581,15 @@ def trigger_push_changes(
     if group_id != group.id:
         raise SeerPermissionError("Unknown run id for group")
 
+    analytics.record(
+        AiAutofixPrCreatedStartedEvent(
+            organization_id=group.organization.id,
+            project_id=group.project_id,
+            group_id=group.id,
+            referrer=referrer.value,
+        )
+    )
+
     client.push_changes(
         run_id,
         repo_name=repo_name,

src/sentry/seer/autofix/on_completion_hook.py

@@ -5,11 +5,13 @@
 
 from django.utils import timezone
 
-from sentry import features
+from sentry import analytics, features
+from sentry.analytics.events.autofix_events import AiAutofixPrCreatedCompletedEvent
 from sentry.models.group import Group
 from sentry.models.organization import Organization
 from sentry.models.project import Project
 from sentry.seer.autofix.autofix_agent import (
+    STEP_CONFIGS,
     AutofixStep,
     trigger_autofix_explorer,
     trigger_coding_agent_handoff,
@@ -136,6 +138,8 @@ def _send_step_webhook(
         # to find which step just completed
         webhook_action_type: SeerActionType | None = None
 
+        is_pr_created = False
+
         if current_step is not None:
             artifact = cls.find_latest_artifact_for_step(state, current_step)
             if artifact is not None:
@@ -166,6 +170,15 @@ def _send_step_webhook(
                     }
                     for pull_request in state.repo_pr_states.values()
                 ]
+                is_pr_created = True
+                analytics.record(
+                    AiAutofixPrCreatedCompletedEvent(
+                        organization_id=organization.id,
+                        project_id=group.project_id,
+                        group_id=group.id,
+                        referrer=None if current_referrer is None else current_referrer.value,
+                    )
+                )
             else:
                 webhook_action_type = SeerActionType.CODING_COMPLETED
                 diffs_by_repo = state.get_diffs_by_repo()
@@ -230,11 +243,21 @@ def _send_step_webhook(
                 },
             )
 
-        if current_step is not None:
+        if current_step is not None and not is_pr_created:
             referrer = current_referrer.value if current_referrer is not None else None
             metrics.incr(
                 "autofix.explorer.complete", tags={"step": current_step.value, "referrer": referrer}
             )
+            completed_event_cls = STEP_CONFIGS[current_step].completed_event
+            if completed_event_cls is not None:
+                analytics.record(
+                    completed_event_cls(
+                        organization_id=organization.id,
+                        project_id=group.project_id,
+                        group_id=group.id,
+                        referrer=referrer,
+                    )
+                )
 
     @classmethod
     def _maybe_trigger_supergroups_embedding(