From f98a76a26c890bd774fb4d0b75a275a87e6a2734 Mon Sep 17 00:00:00 2001 From: Hector Date: Thu, 2 Apr 2026 14:18:49 +0100 Subject: [PATCH] feat(autofix): Add analytics events for autofix phase start and completion Record per-phase analytics events when each autofix pipeline phase (root_cause, solution, code_changes) starts and completes. Each phase gets its own event class so they can be queried independently. Events include group_id, referrer, organization_id, and project_id. Started events fire in trigger_autofix_explorer, completed events fire in the on_completion_hook webhook handler. Co-Authored-By: Claude Opus 4.6 Agent transcript: https://claudescope.sentry.dev/share/botteEgndhtvdoXxpJ2frzQPLVabdvt8jNcgx9NaA3k --- src/sentry/analytics/events/__init__.py | 1 + src/sentry/analytics/events/autofix_events.py | 89 +++++++++++++++++++ src/sentry/seer/autofix/autofix_agent.py | 58 ++++++++++++ src/sentry/seer/autofix/on_completion_hook.py | 27 +++++- 4 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 src/sentry/analytics/events/autofix_events.py diff --git a/src/sentry/analytics/events/__init__.py b/src/sentry/analytics/events/__init__.py index b3d4d362bce563..52be4e1b0c1e9a 100644 --- a/src/sentry/analytics/events/__init__.py +++ b/src/sentry/analytics/events/__init__.py @@ -8,6 +8,7 @@ from .api_token_deleted import * # noqa: F401,F403 from .auth_v2 import * # noqa: F401,F403 from .autofix_automation_events import * # noqa: F401,F403 +from .autofix_events import * # noqa: F401,F403 from .checkin_processing_error_stored import * # noqa: F401,F403 from .codeowners_assignment import * # noqa: F401,F403 from .codeowners_created import * # noqa: F401,F403 diff --git a/src/sentry/analytics/events/autofix_events.py b/src/sentry/analytics/events/autofix_events.py new file mode 100644 index 00000000000000..09e52c9acc5385 --- /dev/null +++ b/src/sentry/analytics/events/autofix_events.py @@ -0,0 +1,89 @@ +from sentry import analytics + + +@analytics.eventclass() +class AiAutofixPhaseEvent(analytics.Event): + organization_id: int + project_id: int + group_id: int + referrer: str | None + + +@analytics.eventclass("ai.autofix.root_cause.started") +class AiAutofixRootCauseStartedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.solution.started") +class AiAutofixSolutionStartedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.code_changes.started") +class AiAutofixCodeChangesStartedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.root_cause.completed") +class AiAutofixRootCauseCompletedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.solution.completed") +class AiAutofixSolutionCompletedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.code_changes.completed") +class AiAutofixCodeChangesCompletedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.impact_assessment.started") +class AiAutofixImpactAssessmentStartedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.impact_assessment.completed") +class AiAutofixImpactAssessmentCompletedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.triage.started") +class AiAutofixTriageStartedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.triage.completed") +class AiAutofixTriageCompletedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.pr_created.started") +class AiAutofixPrCreatedStartedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.pr_created.completed") +class AiAutofixPrCreatedCompletedEvent(AiAutofixPhaseEvent): + pass + + +@analytics.eventclass("ai.autofix.agent_handoff") +class AiAutofixAgentHandoffEvent(AiAutofixPhaseEvent): + pass + + +analytics.register(AiAutofixRootCauseStartedEvent) +analytics.register(AiAutofixSolutionStartedEvent) +analytics.register(AiAutofixCodeChangesStartedEvent) +analytics.register(AiAutofixRootCauseCompletedEvent) +analytics.register(AiAutofixSolutionCompletedEvent) +analytics.register(AiAutofixCodeChangesCompletedEvent) +analytics.register(AiAutofixImpactAssessmentStartedEvent) +analytics.register(AiAutofixImpactAssessmentCompletedEvent) +analytics.register(AiAutofixTriageStartedEvent) +analytics.register(AiAutofixTriageCompletedEvent) +analytics.register(AiAutofixPrCreatedStartedEvent) +analytics.register(AiAutofixPrCreatedCompletedEvent) +analytics.register(AiAutofixAgentHandoffEvent) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 2e0898cc375f7c..98b6ae7a649de2 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -8,6 +8,22 @@ from pydantic import BaseModel from rest_framework.exceptions import PermissionDenied +from sentry import analytics +from sentry.analytics.events.autofix_events import ( + AiAutofixAgentHandoffEvent, + AiAutofixCodeChangesCompletedEvent, + AiAutofixCodeChangesStartedEvent, + AiAutofixImpactAssessmentCompletedEvent, + AiAutofixImpactAssessmentStartedEvent, + AiAutofixPhaseEvent, + AiAutofixPrCreatedStartedEvent, + AiAutofixRootCauseCompletedEvent, + AiAutofixRootCauseStartedEvent, + AiAutofixSolutionCompletedEvent, + AiAutofixSolutionStartedEvent, + AiAutofixTriageCompletedEvent, + AiAutofixTriageStartedEvent, +) from sentry.constants import ENABLE_SEER_CODING_DEFAULT from sentry.seer.autofix.artifact_schemas import ( ImpactAssessmentArtifact, @@ -82,10 +98,14 @@ def __init__( artifact_schema: type[BaseModel] | None, prompt_fn: Callable[..., str], enable_coding: bool = False, + started_event: type[AiAutofixPhaseEvent] | None = None, + completed_event: type[AiAutofixPhaseEvent] | None = None, ): self.artifact_schema = artifact_schema self.prompt_fn = prompt_fn self.enable_coding = enable_coding + self.started_event = started_event + self.completed_event = completed_event # Step configurations mapping step to its artifact schema and prompt @@ -93,23 +113,33 @@ def __init__( AutofixStep.ROOT_CAUSE: StepConfig( artifact_schema=RootCauseArtifact, prompt_fn=root_cause_prompt, + started_event=AiAutofixRootCauseStartedEvent, + completed_event=AiAutofixRootCauseCompletedEvent, ), AutofixStep.SOLUTION: StepConfig( artifact_schema=SolutionArtifact, prompt_fn=solution_prompt, + started_event=AiAutofixSolutionStartedEvent, + completed_event=AiAutofixSolutionCompletedEvent, ), AutofixStep.CODE_CHANGES: StepConfig( artifact_schema=None, # Code changes read from file_patches prompt_fn=code_changes_prompt, enable_coding=True, + started_event=AiAutofixCodeChangesStartedEvent, + completed_event=AiAutofixCodeChangesCompletedEvent, ), AutofixStep.IMPACT_ASSESSMENT: StepConfig( artifact_schema=ImpactAssessmentArtifact, prompt_fn=impact_assessment_prompt, + started_event=AiAutofixImpactAssessmentStartedEvent, + completed_event=AiAutofixImpactAssessmentCompletedEvent, ), AutofixStep.TRIAGE: StepConfig( artifact_schema=TriageArtifact, prompt_fn=triage_prompt, + started_event=AiAutofixTriageStartedEvent, + completed_event=AiAutofixTriageCompletedEvent, ), } @@ -215,6 +245,16 @@ def trigger_autofix_explorer( """ config = STEP_CONFIGS[step] + + if config.started_event is not None: + analytics.record( + config.started_event( + organization_id=group.organization.id, + project_id=group.project_id, + group_id=group.id, + referrer=referrer.value, + ) + ) client = get_autofix_explorer_client( group, intelligence_level=intelligence_level, @@ -500,6 +540,15 @@ def trigger_coding_agent_handoff( auto_create_pr=auto_create_pr, ) + analytics.record( + AiAutofixAgentHandoffEvent( + organization_id=group.organization.id, + project_id=group.project_id, + group_id=group.id, + referrer=referrer.value, + ) + ) + metrics.incr( "autofix.explorer.trigger", tags={"step": "coding_agent_handoff", "referrer": referrer.value}, @@ -532,6 +581,15 @@ def trigger_push_changes( if group_id != group.id: raise SeerPermissionError("Unknown run id for group") + analytics.record( + AiAutofixPrCreatedStartedEvent( + organization_id=group.organization.id, + project_id=group.project_id, + group_id=group.id, + referrer=referrer.value, + ) + ) + client.push_changes( run_id, repo_name=repo_name, diff --git a/src/sentry/seer/autofix/on_completion_hook.py b/src/sentry/seer/autofix/on_completion_hook.py index 0f86910bb20719..b392eedd63f53c 100644 --- a/src/sentry/seer/autofix/on_completion_hook.py +++ b/src/sentry/seer/autofix/on_completion_hook.py @@ -5,11 +5,13 @@ from django.utils import timezone -from sentry import features +from sentry import analytics, features +from sentry.analytics.events.autofix_events import AiAutofixPrCreatedCompletedEvent from sentry.models.group import Group from sentry.models.organization import Organization from sentry.models.project import Project from sentry.seer.autofix.autofix_agent import ( + STEP_CONFIGS, AutofixStep, trigger_autofix_explorer, trigger_coding_agent_handoff, @@ -136,6 +138,8 @@ def _send_step_webhook( # to find which step just completed webhook_action_type: SeerActionType | None = None + is_pr_created = False + if current_step is not None: artifact = cls.find_latest_artifact_for_step(state, current_step) if artifact is not None: @@ -166,6 +170,15 @@ def _send_step_webhook( } for pull_request in state.repo_pr_states.values() ] + is_pr_created = True + analytics.record( + AiAutofixPrCreatedCompletedEvent( + organization_id=organization.id, + project_id=group.project_id, + group_id=group.id, + referrer=None if current_referrer is None else current_referrer.value, + ) + ) else: webhook_action_type = SeerActionType.CODING_COMPLETED diffs_by_repo = state.get_diffs_by_repo() @@ -230,11 +243,21 @@ def _send_step_webhook( }, ) - if current_step is not None: + if current_step is not None and not is_pr_created: referrer = current_referrer.value if current_referrer is not None else None metrics.incr( "autofix.explorer.complete", tags={"step": current_step.value, "referrer": referrer} ) + completed_event_cls = STEP_CONFIGS[current_step].completed_event + if completed_event_cls is not None: + analytics.record( + completed_event_cls( + organization_id=organization.id, + project_id=group.project_id, + group_id=group.id, + referrer=referrer, + ) + ) @classmethod def _maybe_trigger_supergroups_embedding(