feat(dashboards): Track metrics for Seer dashboard create and edit flows (#112595)

- Adds `dashboards.seer.create.save` metric when a user saves a
Seer-generated dashboard for the first time
- Adds `dashboards.seer.edit.save` metric when a user saves AI-suggested
changes on an existing dashboard
- Adds `source` attribute (`create` | `edit`) to
`dashboards.seer.validation` metric to distinguish between flows
- Refactors `validateDashboardAndRecordMetrics` into shared
`createFromSeerUtils` and threads `seerRunId` through the edit flow
callbacks

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: edwardgou-sentry

static/app/views/dashboards/createFromSeer.tsx

@@ -4,18 +4,16 @@ import * as Sentry from '@sentry/react';
 import {Alert} from '@sentry/scraps/alert';
 import {Stack} from '@sentry/scraps/layout';
 
-import {validateDashboard} from 'sentry/actionCreators/dashboards';
 import {addErrorMessage} from 'sentry/actionCreators/indicator';
 import {ErrorBoundary} from 'sentry/components/errorBoundary';
 import {t} from 'sentry/locale';
-import type {Organization} from 'sentry/types/organization';
 import {useLocation} from 'sentry/utils/useLocation';
 import {useOrganization} from 'sentry/utils/useOrganization';
 import {CreateFromSeerLoading} from 'sentry/views/dashboards/createFromSeerLoading';
 import {CreateFromSeerPrompt} from 'sentry/views/dashboards/createFromSeerPrompt';
 
 import {WidgetErrorProvider} from './contexts/widgetErrorContext';
-import {statusIsTerminal} from './createFromSeerUtils';
+import {statusIsTerminal, validateDashboardAndRecordMetrics} from './createFromSeerUtils';
 import {DashboardChatPanel, type WidgetError} from './dashboardChatPanel';
 import {EMPTY_DASHBOARD} from './data';
 import {DashboardDetailWithInjectedProps as DashboardDetail} from './detail';
@@ -25,34 +23,6 @@ import {useSeerDashboardSession} from './useSeerDashboardSession';
 
 const EMPTY_DASHBOARDS: never[] = [];
 
-async function validateDashboardAndRecordMetrics(
-  organization: Organization,
-  newDashboard: DashboardDetails,
-  seerRunId: number | null
-) {
-  try {
-    await validateDashboard(organization.slug, newDashboard);
-    Sentry.metrics.count('dashboards.seer.validation', 1, {
-      attributes: {
-        status: 'success',
-        organization_slug: organization.slug,
-        ...(seerRunId ? {seer_run_id: seerRunId} : {}),
-      },
-    });
-  } catch (error) {
-    Sentry.metrics.count('dashboards.seer.validation', 1, {
-      attributes: {
-        status: 'failure',
-        organization_slug: organization.slug,
-        ...(seerRunId ? {seer_run_id: seerRunId} : {}),
-      },
-    });
-    Sentry.captureException(error, {
-      tags: {seer_run_id: seerRunId},
-    });
-  }
-}
-
 export default function CreateFromSeer() {
   const organization = useOrganization();
   const location = useLocation();
@@ -89,7 +59,12 @@ export default function CreateFromSeer() {
   const handlePostCompletePollEnd = useCallback(() => {
     if (!hasValidatedRef.current && hasSeenNonTerminalRef.current) {
       hasValidatedRef.current = true;
-      validateDashboardAndRecordMetrics(organization, dashboard, seerRunId);
+      validateDashboardAndRecordMetrics({
+        organization,
+        dashboard,
+        seerRunId,
+        source: 'create',
+      });
     }
   }, [organization, dashboard, seerRunId]);
 

static/app/views/dashboards/createFromSeerUtils.tsx

@@ -1,3 +1,7 @@
+import * as Sentry from '@sentry/react';
+
+import {validateDashboard} from 'sentry/actionCreators/dashboards';
+import type {Organization} from 'sentry/types/organization';
 import type {SeerExplorerResponse} from 'sentry/views/seerExplorer/hooks/useSeerExplorer';
 
 import {
@@ -8,7 +12,7 @@ import {
   getInitialColumnDepths,
 } from './layoutUtils';
 import {DEFAULT_TABLE_LIMIT} from './types';
-import type {Widget, WidgetLayout} from './types';
+import type {DashboardDetails, Widget, WidgetLayout} from './types';
 
 const DASHBOARD_ARTIFACT_KEY = 'dashboard';
 
@@ -128,3 +132,39 @@ export function extractDashboardFromSession(
 export function statusIsTerminal(status?: string | null) {
   return status === 'completed' || status === 'error' || status === 'awaiting_user_input';
 }
+
+export async function validateDashboardAndRecordMetrics({
+  organization,
+  dashboard,
+  seerRunId,
+  source,
+}: {
+  dashboard: DashboardDetails;
+  organization: Organization;
+  seerRunId: number | null;
+  source: 'create' | 'edit';
+}) {
+  try {
+    await validateDashboard(organization.slug, dashboard);
+    Sentry.metrics.count('dashboards.seer.validation', 1, {
+      attributes: {
+        status: 'success',
+        organization_slug: organization.slug,
+        ...(seerRunId ? {seer_run_id: seerRunId} : {}),
+        source,
+      },
+    });
+  } catch (error) {
+    Sentry.metrics.count('dashboards.seer.validation', 1, {
+      attributes: {
+        status: 'failure',
+        organization_slug: organization.slug,
+        ...(seerRunId ? {seer_run_id: seerRunId} : {}),
+        source,
+      },
+    });
+    Sentry.captureException(error, {
+      tags: {seer_run_id: seerRunId},
+    });
+  }
+}

static/app/views/dashboards/dashboardChatPanel.tsx

@@ -11,7 +11,6 @@ import {Container, Flex, Stack} from '@sentry/scraps/layout';
 import {IconChevron, IconSeer} from 'sentry/icons';
 import {t} from 'sentry/locale';
 import {MarkedText} from 'sentry/utils/marked/markedText';
-import {useLocation} from 'sentry/utils/useLocation';
 import {BlockComponent} from 'sentry/views/seerExplorer/blockComponents';
 import type {PendingUserInput} from 'sentry/views/seerExplorer/hooks/useSeerExplorer';
 import type {Block} from 'sentry/views/seerExplorer/types';
@@ -41,14 +40,10 @@ export function DashboardChatPanel({
   widgetErrors,
 }: DashboardChatPanelProps) {
   const theme = useTheme();
-  const location = useLocation();
   const [inputValue, setInputValue] = useState('');
   const [isHistoryExpanded, setIsHistoryExpanded] = useState(true);
   const textAreaRef = useRef<HTMLTextAreaElement>(null);
   const chatContainerRef = useRef<HTMLDivElement>(null);
-  const seerRunId = location.query?.seerRunId
-    ? Number(location.query.seerRunId)
-    : undefined;
 
   // Expand history automatically when updating triggered by user input
   useEffect(() => {
@@ -136,7 +131,6 @@ export function DashboardChatPanel({
           ref={chatContainerRef}
           blocks={blocks}
           pendingUserInput={pendingUserInput}
-          seerRunId={seerRunId}
           isError={isError}
           widgetErrors={widgetErrors}
         />
@@ -167,15 +161,13 @@ const ChatHistory = memo(function ChatHistoryInner({
   ref,
   blocks,
   pendingUserInput,
-  seerRunId,
   isError,
   widgetErrors,
 }: {
   blocks: Block[];
   ref: React.Ref<HTMLDivElement>;
   isError?: boolean;
   pendingUserInput?: PendingUserInput | null;
-  seerRunId?: number;
   widgetErrors?: WidgetError[];
 }) {
   return (
@@ -188,12 +180,7 @@ const ChatHistory = memo(function ChatHistoryInner({
     >
       <Stack>
         {blocks.map((block, index) => (
-          <BlockComponent
-            key={block.id}
-            block={block}
-            blockIndex={index}
-            runId={seerRunId}
-          />
+          <BlockComponent key={block.id} block={block} blockIndex={index} />
         ))}
         {pendingUserInput && pendingUserInput.data.questions?.length > 0 && (
           <ChatMessageContainer padding="xl">

static/app/views/dashboards/dashboardEditSeerChat.tsx

@@ -8,7 +8,10 @@ import {useSeerDashboardSession} from './useSeerDashboardSession';
 
 interface DashboardEditSeerChatProps {
   dashboard: DashboardDetails;
-  onDashboardUpdate: (dashboard: Pick<DashboardDetails, 'title' | 'widgets'>) => void;
+  onDashboardUpdate: (
+    dashboard: Pick<DashboardDetails, 'title' | 'widgets'>,
+    seerRunId: number | null
+  ) => void;
 }
 
 export function DashboardEditSeerChat({
@@ -22,8 +25,8 @@ export function DashboardEditSeerChat({
     organization.features.includes('dashboards-ai-generate');
 
   const handleDashboardUpdate = useCallback(
-    (data: {title: string; widgets: Widget[]}) => {
-      onDashboardUpdate({title: data.title, widgets: data.widgets});
+    (data: {title: string; widgets: Widget[]}, seerRunId: number | null) => {
+      onDashboardUpdate({title: data.title, widgets: data.widgets}, seerRunId);
     },
     [onDashboardUpdate]
   );

static/app/views/dashboards/detail.tsx

@@ -86,6 +86,7 @@ import {DiscoverQueryPageSource} from 'sentry/views/performance/utils';
 
 import {PrebuiltDashboardOnboardingGate} from './components/prebuiltDashboardOnboardingGate';
 import {Controls} from './controls';
+import {validateDashboardAndRecordMetrics} from './createFromSeerUtils';
 import {Dashboard} from './dashboard';
 import {DashboardEditSeerChat} from './dashboardEditSeerChat';
 import {DEFAULT_STATS_PERIOD} from './data';
@@ -158,6 +159,8 @@ type State = {
   isSavingDashboardFilters: boolean;
   isWidgetBuilderOpen: boolean;
   modifiedDashboard: DashboardDetails | null;
+  seerEditApplied: boolean;
+  seerRunId: number | null;
   widgetLegendState: WidgetLegendSelectionState;
   widgetLimitReached: boolean;
   newlyAddedWidget?: Widget;
@@ -214,6 +217,8 @@ class DashboardDetail extends Component<Props, State> {
     openWidgetTemplates: undefined,
     newlyAddedWidget: undefined,
     isCommittingChanges: false,
+    seerEditApplied: false,
+    seerRunId: null,
   };
 
   componentDidMount() {
@@ -856,6 +861,18 @@ class DashboardDetail extends Component<Props, State> {
             (newDashboard: DashboardDetails) => {
               addSuccessMessage(t('Dashboard created'));
               trackAnalytics('dashboards2.create.complete', {organization});
+              const seerRunId = location.query?.seerRunId
+                ? Number(location.query.seerRunId)
+                : null;
+              if (seerRunId) {
+                Sentry.metrics.count('dashboards.seer.create.save', 1, {
+                  attributes: {
+                    organization_slug: organization.slug,
+                    dashboard_id: newDashboard.id,
+                    seer_run_id: seerRunId,
+                  },
+                });
+              }
               this.setState(
                 {
                   dashboardState: DashboardState.VIEW,
@@ -899,6 +916,8 @@ class DashboardDetail extends Component<Props, State> {
             this.setState({
               dashboardState: DashboardState.VIEW,
               modifiedDashboard: null,
+              seerEditApplied: false,
+              seerRunId: null,
             });
             return;
           }
@@ -912,11 +931,22 @@ class DashboardDetail extends Component<Props, State> {
               }
               addSuccessMessage(t('Dashboard updated'));
               trackAnalytics('dashboards2.edit.complete', {organization});
+              if (this.state.seerEditApplied && this.state.seerRunId) {
+                Sentry.metrics.count('dashboards.seer.edit.save', 1, {
+                  attributes: {
+                    organization_slug: organization.slug,
+                    dashboard_id: newDashboard.id,
+                    seer_run_id: this.state.seerRunId,
+                  },
+                });
+              }
               this.setState(
                 {
                   dashboardState: DashboardState.VIEW,
                   modifiedDashboard: null,
                   isCommittingChanges: false,
+                  seerEditApplied: false,
+                  seerRunId: null,
                 },
                 () => {
                   if (dashboard && newDashboard.id !== dashboard.id) {
@@ -941,6 +971,8 @@ class DashboardDetail extends Component<Props, State> {
         this.setState({
           dashboardState: DashboardState.VIEW,
           modifiedDashboard: null,
+          seerEditApplied: false,
+          seerRunId: null,
         });
         break;
       }
@@ -949,6 +981,8 @@ class DashboardDetail extends Component<Props, State> {
         this.setState({
           dashboardState: DashboardState.VIEW,
           modifiedDashboard: null,
+          seerEditApplied: false,
+          seerRunId: null,
         });
         break;
       }
@@ -961,21 +995,37 @@ class DashboardDetail extends Component<Props, State> {
     });
   };
 
-  handleSeerDashboardUpdate = ({
-    title,
-    widgets,
-  }: Pick<DashboardDetails, 'title' | 'widgets'>) => {
-    this.setState(state => {
-      const dashboard = cloneDashboard(state.modifiedDashboard ?? this.props.dashboard);
-      return {
-        widgetLimitReached: widgets.length >= MAX_WIDGETS,
-        modifiedDashboard: {
+  handleSeerDashboardUpdate = (
+    {title, widgets}: Pick<DashboardDetails, 'title' | 'widgets'>,
+    seerRunId: number | null
+  ) => {
+    const {organization} = this.props;
+    this.setState(
+      state => {
+        const dashboard = cloneDashboard(state.modifiedDashboard ?? this.props.dashboard);
+        const updatedDashboard = {
           ...dashboard,
           widgets,
           ...(title === undefined ? {} : {title}),
-        },
-      };
-    });
+        };
+        return {
+          widgetLimitReached: widgets.length >= MAX_WIDGETS,
+          seerEditApplied: true,
+          seerRunId,
+          modifiedDashboard: updatedDashboard,
+        };
+      },
+      () => {
+        if (this.state.modifiedDashboard) {
+          validateDashboardAndRecordMetrics({
+            organization,
+            dashboard: this.state.modifiedDashboard,
+            seerRunId,
+            source: 'edit',
+          });
+        }
+      }
+    );
   };
 
   handleUpdateEditStateWidgets = (widgets: Widget[]) => {

static/app/views/dashboards/useSeerDashboardSession.tsx

@@ -38,7 +38,10 @@ async function startDashboardEditSession(
 }
 
 interface UseSeerDashboardSessionOptions {
-  onDashboardUpdate: (data: {title: string; widgets: Widget[]}) => void;
+  onDashboardUpdate: (
+    data: {title: string; widgets: Widget[]},
+    seerRunId: number | null
+  ) => void;
   dashboard?: Pick<DashboardDetails, 'title' | 'widgets'>;
   enabled?: boolean;
   onPostCompletePollEnd?: () => void;
@@ -127,10 +130,17 @@ export function useSeerDashboardSession({
       }
       const dashboardData = extractDashboardFromSession(session);
       if (dashboardData) {
-        onDashboardUpdate(dashboardData);
+        onDashboardUpdate(dashboardData, seerRunId);
       }
     }
-  }, [isUpdating, sessionStatus, session, sessionUpdatedAt, onDashboardUpdate]);
+  }, [
+    isUpdating,
+    sessionStatus,
+    session,
+    sessionUpdatedAt,
+    onDashboardUpdate,
+    seerRunId,
+  ]);
 
   const sendFollowUpMessage = useCallback(
     async (message: string) => {