fix(webhooks): New task for metric alerts to fire in process (#112234)

Author: Christinarlong

src/sentry/incidents/action_handlers.py

@@ -462,22 +462,15 @@ def send_alert(
 
         incident_serialized_response = serialize(incident, serializer=IncidentSerializer())
 
-        success = send_incident_alert_notification(
+        send_incident_alert_notification(
             notification_context=notification_context,
             alert_context=alert_context,
             metric_issue_context=metric_issue_context,
             incident_serialized_response=incident_serialized_response,
             organization=incident.organization,
+            project_id=project.id,
             notification_uuid=notification_uuid,
         )
-        if success:
-            self.record_alert_sent_analytics(
-                organization_id=incident.organization.id,
-                project_id=project.id,
-                alert_id=incident.alert_rule.id,
-                external_id=action.sentry_app_id,
-                notification_uuid=notification_uuid,
-            )
 
 
 def format_duration(minutes):

src/sentry/integrations/services/integration/impl.py

@@ -45,7 +45,6 @@
     serialize_integration_external_project,
     serialize_organization_integration,
 )
-from sentry.rules.actions.notify_event_service import find_alert_rule_action_ui_component
 from sentry.sentry_apps.api.serializers.app_platform_event import AppPlatformEvent
 from sentry.sentry_apps.metrics import (
     SentryAppEventType,
@@ -54,7 +53,11 @@
 )
 from sentry.sentry_apps.models.sentry_app import SentryApp
 from sentry.sentry_apps.models.sentry_app_installation import SentryAppInstallation
-from sentry.sentry_apps.utils.webhooks import MetricAlertActionType, SentryAppResourceType
+from sentry.sentry_apps.utils.webhooks import (
+    MetricAlertActionType,
+    SentryAppResourceType,
+    find_alert_rule_action_ui_component,
+)
 from sentry.shared_integrations.exceptions import ApiError
 from sentry.utils import json
 from sentry.utils.sentry_apps import send_and_save_webhook_request

src/sentry/notifications/notification_action/metric_alert_registry/handlers/sentry_app_metric_alert_handler.py

@@ -54,5 +54,6 @@ def send_alert(
             metric_issue_context=metric_issue_context,
             incident_serialized_response=incident_serialized_response,
             organization=organization,
+            project_id=project.id,
             notification_uuid=notification_uuid,
         )

src/sentry/rules/actions/notify_event_service.py

@@ -13,15 +13,13 @@
     NotificationContext,
 )
 from sentry.integrations.metric_alerts import incident_attachment_info
-from sentry.integrations.services.integration import integration_service
 from sentry.models.organization import Organization
 from sentry.plugins.base import plugins
 from sentry.rules.actions.base import EventAction
 from sentry.rules.actions.services import PluginService
 from sentry.rules.base import CallbackFuture
-from sentry.sentry_apps.api.serializers.app_platform_event import AppPlatformEvent
 from sentry.sentry_apps.services.app import RpcSentryAppService, app_service
-from sentry.sentry_apps.tasks.sentry_apps import notify_sentry_app
+from sentry.sentry_apps.tasks.sentry_apps import notify_sentry_app, send_metric_alert_webhook
 from sentry.services.eventstore.models import GroupEvent
 from sentry.utils import json, metrics
 from sentry.utils.forms import set_field_choices
@@ -64,15 +62,15 @@ def send_incident_alert_notification(
     metric_issue_context: MetricIssueContext,
     incident_serialized_response: IncidentSerializerResponse,
     organization: Organization,
+    project_id: int,
     notification_uuid: str | None = None,
-) -> bool:
+) -> None:
     """
     When a metric alert is triggered, send incident data to the SentryApp's webhook.
     :param action: The triggered `AlertRuleTriggerAction`.
     :param incident: The `Incident` for which to build a payload.
-    :param metric_value: The value of the metric that triggered this alert to
-    fire.
-    :return:
+    :param metric_value: The value of the metric that triggered this alert to fire.
+    :param project_id: project id will be used for analytics after sending the webhook.
     """
     incident_attachment = build_incident_attachment(
         alert_context,
@@ -85,40 +83,16 @@ def send_incident_alert_notification(
     if notification_context.sentry_app_id is None:
         raise ValueError("Sentry app ID is required")
 
-    success = integration_service.send_incident_alert_notification(
-        sentry_app_id=notification_context.sentry_app_id,
+    send_metric_alert_webhook.delay(
+        sentry_app_id=int(notification_context.sentry_app_id),
         new_status=metric_issue_context.new_status.value,
         incident_attachment_json=json.dumps(incident_attachment),
         organization_id=organization.id,
-        # TODO(iamrajjoshi): The rest of the params are unused
-        action_id=-1,
-        incident_id=-1,
-        metric_value=-1,
-    )
-    return success
-
-
-def find_alert_rule_action_ui_component(app_platform_event: AppPlatformEvent) -> bool:
-    """
-    Loop through the triggers for the alert rule event. For each trigger, check
-    if an action is an alert rule UI Component
-    """
-    triggers = (
-        getattr(app_platform_event, "data", {})
-        .get("metric_alert", {})
-        .get("alert_rule", {})
-        .get("triggers", [])
+        project_id=project_id,
+        alert_id=alert_context.action_identifier_id,
+        notification_uuid=notification_uuid,
     )
 
-    actions = [
-        action
-        for trigger in triggers
-        for action in trigger.get("actions", {})
-        if (action.get("type") == "sentry_app" and action.get("settings") is not None)
-    ]
-
-    return bool(len(actions))
-
 
 class NotifyEventServiceForm(forms.Form):
     service = forms.ChoiceField(choices=())

src/sentry/sentry_apps/metrics.py

@@ -60,6 +60,7 @@ class SentryAppWebhookFailureReason(StrEnum):
     EVENT_NOT_IN_SERVCEHOOK = "event_not_in_servicehook"
     MISSING_ISSUE_OCCURRENCE = "missing_issue_occurrence"
     MISSING_USER = "missing_user"
+    MULTIPLE_INSTALLATIONS = "multiple_installations"
 
 
 class SentryAppWebhookHaltReason(StrEnum):

src/sentry/sentry_apps/tasks/__init__.py

@@ -8,6 +8,7 @@
     regenerate_service_hooks_for_installation,
     send_alert_webhook,
     send_alert_webhook_v2,
+    send_metric_alert_webhook,
     send_resource_change_webhook,
     workflow_notification,
 )
@@ -26,4 +27,5 @@
     "send_alert_webhook_v2",
     "send_resource_change_webhook",
     "workflow_notification",
+    "send_metric_alert_webhook",
 )

src/sentry/sentry_apps/tasks/sentry_apps.py

@@ -17,6 +17,7 @@
 from sentry.analytics.events.alert_rule_ui_component_webhook_sent import (
     AlertRuleUiComponentWebhookSentEvent,
 )
+from sentry.analytics.events.alert_sent import AlertSentEvent
 from sentry.analytics.events.comment_webhooks import (
     CommentCreatedEvent,
     CommentDeletedEvent,
@@ -36,6 +37,7 @@
 from sentry.db.models.base import Model
 from sentry.exceptions import RestrictedIPAddress
 from sentry.hybridcloud.rpc.caching import cell_caching_service
+from sentry.incidents.models.incident import INCIDENT_STATUS, IncidentStatus
 from sentry.issues.issue_occurrence import IssueOccurrence
 from sentry.models.activity import Activity
 from sentry.models.group import Group
@@ -63,7 +65,12 @@
 )
 from sentry.sentry_apps.services.hook.service import hook_service
 from sentry.sentry_apps.utils.errors import SentryAppSentryError
-from sentry.sentry_apps.utils.webhooks import IssueAlertActionType, SentryAppResourceType
+from sentry.sentry_apps.utils.webhooks import (
+    IssueAlertActionType,
+    MetricAlertActionType,
+    SentryAppResourceType,
+    find_alert_rule_action_ui_component,
+)
 from sentry.services.eventstore.models import BaseEvent, Event, GroupEvent
 from sentry.shared_integrations.exceptions import ApiHostError, ApiTimeoutError, ClientError
 from sentry.silo.base import SiloMode
@@ -72,7 +79,7 @@
 from sentry.types.rules import RuleFuture
 from sentry.users.services.user.model import RpcUser
 from sentry.users.services.user.service import user_service
-from sentry.utils import metrics
+from sentry.utils import json, metrics
 from sentry.utils.function_cache import cache_func_for_models
 from sentry.utils.http import absolute_uri
 from sentry.utils.sentry_apps import send_and_save_webhook_request
@@ -908,6 +915,113 @@ def regenerate_service_hooks_for_installation(
             )
 
 
+def _record_metric_alert_sent_analytics(
+    organization_id: int,
+    project_id: int,
+    alert_id: int,
+    sentry_app_id: int,
+    notification_uuid: str | None,
+) -> None:
+    try:
+        analytics.record(
+            AlertSentEvent(
+                organization_id=organization_id,
+                project_id=project_id,
+                alert_id=alert_id,
+                alert_type="metric_alert",
+                provider="sentry_app",
+                external_id=str(sentry_app_id),
+                notification_uuid=notification_uuid,
+            )
+        )
+    except Exception as e:
+        sentry_sdk.capture_exception(e)
+
+
+def _record_metric_alert_ui_component_analytics(
+    organization_id: int,
+    sentry_app_id: int,
+    app_platform_event: AppPlatformEvent,
+) -> None:
+    if not find_alert_rule_action_ui_component(app_platform_event):
+        return
+    try:
+        analytics.record(
+            AlertRuleUiComponentWebhookSentEvent(
+                organization_id=organization_id,
+                sentry_app_id=sentry_app_id,
+                event=f"{app_platform_event.resource}.{app_platform_event.action}",
+            )
+        )
+    except Exception as e:
+        sentry_sdk.capture_exception(e)
+
+
+@instrumented_task(
+    name="sentry.sentry_apps.tasks.sentry_apps.send_metric_alert_webhook",
+    namespace=sentryapp_tasks,
+    retry=Retry(times=3, delay=60 * 5),
+    silo_mode=SiloMode.CELL,
+)
+@retry_decorator
+def send_metric_alert_webhook(
+    sentry_app_id: int,
+    new_status: int,
+    incident_attachment_json: str,
+    organization_id: int,
+    project_id: int,
+    alert_id: int,
+    notification_uuid: str | None = None,
+    **kwargs: Any,
+) -> None:
+    try:
+        new_status_str = INCIDENT_STATUS[IncidentStatus(new_status)].lower()
+        event = SentryAppEventType(
+            f"{SentryAppResourceType.METRIC_ALERT}.{MetricAlertActionType(new_status_str)}"
+        )
+    except ValueError as e:
+        sentry_sdk.capture_exception(e)
+        return
+
+    with SentryAppInteractionEvent(
+        operation_type=SentryAppInteractionType.PREPARE_WEBHOOK,
+        event_type=event,
+    ).capture() as lifecycle:
+        sentry_app = app_service.get_sentry_app_by_id(id=sentry_app_id)
+        if sentry_app is None:
+            lifecycle.record_failure(SentryAppWebhookFailureReason.MISSING_SENTRY_APP)
+            return
+
+        installations = app_service.get_many(
+            filter=dict(
+                organization_id=organization_id,
+                app_ids=[sentry_app.id],
+                status=SentryAppInstallationStatus.INSTALLED,
+            )
+        )
+        if not installations:
+            lifecycle.record_failure(SentryAppWebhookFailureReason.MISSING_INSTALLATION)
+            return
+
+        if len(installations) > 1:
+            lifecycle.record_failure(SentryAppWebhookFailureReason.MULTIPLE_INSTALLATIONS)
+            return
+
+        app_platform_event = AppPlatformEvent(
+            resource=SentryAppResourceType.METRIC_ALERT,
+            action=MetricAlertActionType(new_status_str),
+            install=installations[0],
+            data=json.loads(incident_attachment_json),
+        )
+
+    send_and_save_webhook_request(sentry_app, app_platform_event)
+
+    _record_metric_alert_sent_analytics(
+        organization_id, project_id, alert_id, sentry_app.id, notification_uuid
+    )
+    _record_metric_alert_ui_component_analytics(organization_id, sentry_app.id, app_platform_event)
+
+
 @instrumented_task(
     name="sentry.sentry_apps.tasks.sentry_apps.broadcast_webhooks_for_organization",
     namespace=sentryapp_tasks,

src/sentry/sentry_apps/utils/webhooks.py

@@ -1,5 +1,10 @@
+from __future__ import annotations
+
 from enum import StrEnum
-from typing import Final
+from typing import TYPE_CHECKING, Any, Final
+
+if TYPE_CHECKING:
+    from sentry.sentry_apps.api.serializers.app_platform_event import AppPlatformEvent
 
 
 class SentryAppActionType(StrEnum):
@@ -106,3 +111,25 @@ def map_sentry_app_webhook_events(
 # per-event-type (issue.created, project.deleted, etc.). These are valid
 # resources a Sentry App may subscribe to.
 VALID_EVENT_RESOURCES = EVENT_EXPANSION.keys()
+
+
+def find_alert_rule_action_ui_component(
+    app_platform_event: AppPlatformEvent[dict[str, Any]],
+) -> bool:
+    """
+    Returns True if the metric alert event contains a sentry app action with UI component settings.
+    Used to gate recording of AlertRuleUiComponentWebhookSentEvent analytics.
+    """
+    triggers = (
+        getattr(app_platform_event, "data", {})
+        .get("metric_alert", {})
+        .get("alert_rule", {})
+        .get("triggers", [])
+    )
+    actions = [
+        action
+        for trigger in triggers
+        for action in trigger.get("actions", [])
+        if (action.get("type") == "sentry_app" and action.get("settings") is not None)
+    ]
+    return bool(len(actions))

tests/sentry/notifications/notification_action/metric_alert_registry/test_sentry_app_metric_alert_handler.py

@@ -85,6 +85,7 @@ def test_send_alert(self, mock_send_incident_alert_notification: mock.MagicMock)
             alert_context=alert_context,
             metric_issue_context=metric_issue_context,
             organization=self.detector.project.organization,
+            project_id=self.detector.project.id,
             notification_uuid=notification_uuid,
             incident_serialized_response=get_incident_serializer(self.open_period),
         )