feat(test): add it.isKnownFlake for opt-in stress-testing flaky Jest fixes

.github/workflows/frontend.yml

@@ -159,6 +159,9 @@ jobs:
           #
           # This quiets up the logs quite a bit.
           DEBUG_PRINT_LIMIT: 0
+          # When the "Frontend: Rerun Flaky Tests" label is on the PR,
+          # tests wrapped with it.isKnownFlake() run 50x to validate fixes.
+          RERUN_KNOWN_FLAKY_TESTS: "${{ contains(github.event.pull_request.labels.*.name, 'Frontend: Rerun Flaky Tests') }}"
         run: pnpm run test-ci --forceExit
 
   form-field-registry:

eslint.config.ts

@@ -824,6 +824,10 @@ export default typescript.config([
       'jest/expect-expect': 'off', // Disabled as we have many tests which render as simple validations
       'jest/no-conditional-expect': 'off', // TODO(ryan953): Fix violations then delete this line
       'jest/no-disabled-tests': 'error', // `recommended` set this to warn, we've upgraded to error
+      'jest/no-standalone-expect': [
+        'error',
+        {additionalTestBlockFunctions: ['it.isKnownFlake']},
+      ],
     },
   },
   {

static/app/components/events/eventReplay/index.spec.tsx

@@ -137,14 +137,17 @@ describe('EventReplay', () => {
     });
   });
 
-  it('should render the replay inline onboarding component when replays are enabled and the project supports replay', async () => {
-    MockUseReplayOnboardingSidebarPanel.mockReturnValue({
-      activateSidebar: jest.fn(),
-    });
-    render(<EventReplay {...defaultProps} />, {organization});
-
-    expect(await screen.findByTestId('replay-inline-onboarding')).toBeInTheDocument();
-  });
+  it.isKnownFlake(
+    'should render the replay inline onboarding component when replays are enabled and the project supports replay',
+    async () => {
+      MockUseReplayOnboardingSidebarPanel.mockReturnValue({
+        activateSidebar: jest.fn(),
+      });
+      render(<EventReplay {...defaultProps} />, {organization});
+
+      expect(await screen.findByTestId('replay-inline-onboarding')).toBeInTheDocument();
+    }
+  );
 
   it('should render a replay when there is a replayId from tags', async () => {
     MockUseReplayOnboardingSidebarPanel.mockReturnValue({

static/app/components/groupPreviewTooltip/spanEvidencePreview.spec.tsx

@@ -28,7 +28,7 @@ describe('SpanEvidencePreview', () => {
     expect(mock).not.toHaveBeenCalled();
   });
 
-  it('shows error when request fails', async () => {
+  it.isKnownFlake('shows error when request fails', async () => {
     MockApiClient.addMockResponse({
       url: `/organizations/org-slug/issues/group-id/events/recommended/`,
       body: {},

static/app/components/stackTrace/stackTrace.spec.tsx

@@ -877,7 +877,7 @@ describe('Core StackTrace', () => {
     ).toBeInTheDocument();
   });
 
-  it('shows URL link in tooltip when absPath is an http URL', async () => {
+  it.isKnownFlake('shows URL link in tooltip when absPath is an http URL', async () => {
     jest.useFakeTimers();
     const {event, stacktrace} = makeStackTraceData();
     const frame = stacktrace.frames[stacktrace.frames.length - 1]!;

static/app/components/timeSince.spec.tsx

@@ -58,7 +58,7 @@ describe('TimeSince', () => {
     expect(screen.getByText('10m atrás')).toBeInTheDocument();
   });
 
-  it('respects timezone in tooltip', async () => {
+  it.isKnownFlake('respects timezone in tooltip', async () => {
     const date = new Date('2024-01-15T12:00:00Z');
     render(
       <TimezoneProvider timezone="America/New_York">

static/app/components/versionHoverCard.spec.tsx

@@ -31,7 +31,7 @@ describe('VersionHoverCard', () => {
     });
   });
 
-  it('renders', async () => {
+  it.isKnownFlake('renders', async () => {
     render(
       <VersionHoverCard
         organization={organization}

static/app/gettingStartedDocs/react-native/metrics.spec.tsx

@@ -28,7 +28,7 @@ function renderMockRequests({
 }
 
 describe('getting started with react-native', () => {
-  it('shows React Native metrics onboarding content', async () => {
+  it.isKnownFlake('shows React Native metrics onboarding content', async () => {
     const organization = OrganizationFixture();
     const project = ProjectFixture({platform: 'react-native'});
     renderMockRequests({organization, project});

static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/eventsSearchBar.spec.tsx

@@ -27,7 +27,7 @@ describe('EventsSearchBar', () => {
     });
   });
 
-  it('does not show function tags in has: dropdown', async () => {
+  it.isKnownFlake('does not show function tags in has: dropdown', async () => {
     render(
       <EventsSearchBar
         onClose={jest.fn()}

static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/spansSearchBar.spec.tsx

@@ -122,7 +122,7 @@ describe('SpansSearchBar', () => {
     await screen.findByLabelText('span.op:function');
   });
 
-  it('calls onSearch with the correct query', async () => {
+  it.isKnownFlake('calls onSearch with the correct query', async () => {
     const onSearch = jest.fn();
 
     renderWithProvider({

static/app/views/detectors/list/allMonitors.spec.tsx

@@ -483,81 +483,84 @@ describe('DetectorsList', () => {
       expect(screen.getByRole('button', {name: 'Delete'})).toBeDisabled();
     });
 
-    it('shows option to select all query results when page is selected', async () => {
-      const deleteRequest = MockApiClient.addMockResponse({
-        url: '/organizations/org-slug/detectors/',
-        method: 'DELETE',
-        body: {},
-      });
-
-      render(<AllMonitors />, {organization});
-      renderGlobalModal();
-
-      const testUser = UserFixture({id: '2', email: 'test@example.com'});
-      // Mock the filtered search results - this will be used when search is applied
-      const filteredDetectors = Array.from({length: 20}, (_, i) =>
-        MetricDetectorFixture({
-          id: `filtered-${i}`,
-          name: `Assigned Detector ${i + 1}`,
-          owner: ActorFixture({id: testUser.id, name: testUser.email, type: 'user'}),
-        })
-      );
-
-      MockApiClient.addMockResponse({
-        url: '/organizations/org-slug/detectors/',
-        body: filteredDetectors,
-        headers: {
-          'X-Hits': '50',
-        },
-        match: [
-          MockApiClient.matchQuery({
-            query: '!type:issue_stream assignee:test@example.com',
-          }),
-        ],
-      });
-
-      // Click through menus to select assignee
-      const searchInput = await screen.findByRole('combobox', {
-        name: 'Add a search term',
-      });
-      await userEvent.type(searchInput, 'assignee:test@example.com{enter}');
-
-      // Wait for filtered results to load
-      await screen.findByText('Assigned Detector 1');
-
-      const rows = screen.getAllByTestId('detector-list-row');
-
-      // Focus on first row to make checkbox visible
-      await userEvent.click(rows[0]!);
-      const firstRowCheckbox = within(rows[0]!).getByRole('checkbox');
-      await userEvent.click(firstRowCheckbox);
-      expect(firstRowCheckbox).toBeChecked();
-
-      // Select all on page - master checkbox should now be visible since we have a selection
-      const masterCheckbox = screen.getAllByRole('checkbox')[0]!;
-      await userEvent.click(masterCheckbox);
-
-      // Should show alert with option to select all query results
-      expect(screen.getByText(/20 monitors on this page selected/)).toBeInTheDocument();
-      const selectAllForQuery = screen.getByRole('button', {
-        name: /Select all 50 monitors that match this search query/,
-      });
-      await userEvent.click(selectAllForQuery);
-
-      // Perform an action to verify query-based selection
-      await userEvent.click(screen.getByRole('button', {name: 'Delete'}));
-      const confirmModal = await screen.findByRole('dialog');
-      await userEvent.click(within(confirmModal).getByRole('button', {name: 'Delete'})); // Confirm
-
-      await waitFor(() => {
-        expect(deleteRequest).toHaveBeenCalledWith(
-          '/organizations/org-slug/detectors/',
-          expect.objectContaining({
-            query: {id: undefined, query: 'assignee:test@example.com', project: [1]},
+    it.isKnownFlake(
+      'shows option to select all query results when page is selected',
+      async () => {
+        const deleteRequest = MockApiClient.addMockResponse({
+          url: '/organizations/org-slug/detectors/',
+          method: 'DELETE',
+          body: {},
+        });
+
+        render(<AllMonitors />, {organization});
+        renderGlobalModal();
+
+        const testUser = UserFixture({id: '2', email: 'test@example.com'});
+        // Mock the filtered search results - this will be used when search is applied
+        const filteredDetectors = Array.from({length: 20}, (_, i) =>
+          MetricDetectorFixture({
+            id: `filtered-${i}`,
+            name: `Assigned Detector ${i + 1}`,
+            owner: ActorFixture({id: testUser.id, name: testUser.email, type: 'user'}),
           })
         );
-      });
-    });
+
+        MockApiClient.addMockResponse({
+          url: '/organizations/org-slug/detectors/',
+          body: filteredDetectors,
+          headers: {
+            'X-Hits': '50',
+          },
+          match: [
+            MockApiClient.matchQuery({
+              query: '!type:issue_stream assignee:test@example.com',
+            }),
+          ],
+        });
+
+        // Click through menus to select assignee
+        const searchInput = await screen.findByRole('combobox', {
+          name: 'Add a search term',
+        });
+        await userEvent.type(searchInput, 'assignee:test@example.com{enter}');
+
+        // Wait for filtered results to load
+        await screen.findByText('Assigned Detector 1');
+
+        const rows = screen.getAllByTestId('detector-list-row');
+
+        // Focus on first row to make checkbox visible
+        await userEvent.click(rows[0]!);
+        const firstRowCheckbox = within(rows[0]!).getByRole('checkbox');
+        await userEvent.click(firstRowCheckbox);
+        expect(firstRowCheckbox).toBeChecked();
+
+        // Select all on page - master checkbox should now be visible since we have a selection
+        const masterCheckbox = screen.getAllByRole('checkbox')[0]!;
+        await userEvent.click(masterCheckbox);
+
+        // Should show alert with option to select all query results
+        expect(screen.getByText(/20 monitors on this page selected/)).toBeInTheDocument();
+        const selectAllForQuery = screen.getByRole('button', {
+          name: /Select all 50 monitors that match this search query/,
+        });
+        await userEvent.click(selectAllForQuery);
+
+        // Perform an action to verify query-based selection
+        await userEvent.click(screen.getByRole('button', {name: 'Delete'}));
+        const confirmModal = await screen.findByRole('dialog');
+        await userEvent.click(within(confirmModal).getByRole('button', {name: 'Delete'})); // Confirm
+
+        await waitFor(() => {
+          expect(deleteRequest).toHaveBeenCalledWith(
+            '/organizations/org-slug/detectors/',
+            expect.objectContaining({
+              query: {id: undefined, query: 'assignee:test@example.com', project: [1]},
+            })
+          );
+        });
+      }
+    );
 
     it('disables action buttons when user does not have permissions', async () => {
       const noPermsOrganization = OrganizationFixture({

static/app/views/discover/results/resultsSearchQueryBuilder.spec.tsx

@@ -27,7 +27,7 @@ describe('ResultsSearchQueryBuilder', () => {
     });
   });
 
-  it('does not show function tags in has: dropdown', async () => {
+  it.isKnownFlake('does not show function tags in has: dropdown', async () => {
     render(
       <ResultsSearchQueryBuilder
         query=""
@@ -59,7 +59,7 @@ describe('ResultsSearchQueryBuilder', () => {
     ).not.toBeInTheDocument();
   });
 
-  it('shows normal tags, e.g. transaction, in the dropdown', async () => {
+  it.isKnownFlake('shows normal tags, e.g. transaction, in the dropdown', async () => {
     render(
       <ResultsSearchQueryBuilder
         query=""

static/app/views/explore/metrics/metricsTab.spec.tsx

@@ -700,36 +700,39 @@ describe('MetricsTabContent (tracemetrics-ui-refresh)', () => {
     });
   });
 
-  it('toggles the query builder sidebar with the expand control', async () => {
-    render(
-      <ProviderWrapper>
-        <MetricsTabContent datePageFilterProps={datePageFilterProps} />
-      </ProviderWrapper>,
-      {
-        initialRouterConfig,
-        organization,
-      }
-    );
+  it.isKnownFlake(
+    'toggles the query builder sidebar with the expand control',
+    async () => {
+      render(
+        <ProviderWrapper>
+          <MetricsTabContent datePageFilterProps={datePageFilterProps} />
+        </ProviderWrapper>,
+        {
+          initialRouterConfig,
+          organization,
+        }
+      );
 
-    await waitFor(() => {
-      expect(
-        within(screen.getAllByTestId('metric-toolbar')[0]!).getByRole('button', {
-          name: 'bar',
-        })
-      ).toBeInTheDocument();
-    });
+      await waitFor(() => {
+        expect(
+          within(screen.getAllByTestId('metric-toolbar')[0]!).getByRole('button', {
+            name: 'bar',
+          })
+        ).toBeInTheDocument();
+      });
 
-    expect(screen.getByRole('button', {name: 'Collapse sidebar'})).toBeInTheDocument();
+      expect(screen.getByRole('button', {name: 'Collapse sidebar'})).toBeInTheDocument();
 
-    await userEvent.click(screen.getByRole('button', {name: 'Collapse sidebar'}));
+      await userEvent.click(screen.getByRole('button', {name: 'Collapse sidebar'}));
 
-    expect(screen.queryByTestId('metric-toolbar')).not.toBeInTheDocument();
-    expect(screen.getByRole('button', {name: 'Expand sidebar'})).toBeInTheDocument();
+      expect(screen.queryByTestId('metric-toolbar')).not.toBeInTheDocument();
+      expect(screen.getByRole('button', {name: 'Expand sidebar'})).toBeInTheDocument();
 
-    await userEvent.click(screen.getByRole('button', {name: 'Expand sidebar'}));
+      await userEvent.click(screen.getByRole('button', {name: 'Expand sidebar'}));
 
-    await waitFor(() => {
-      expect(screen.getAllByTestId('metric-toolbar')).toHaveLength(1);
-    });
-  });
+      await waitFor(() => {
+        expect(screen.getAllByTestId('metric-toolbar')).toHaveLength(1);
+      });
+    }
+  );
 });

static/app/views/issueDetails/groupReplays/useReplaysFromIssue.spec.tsx

@@ -21,7 +21,7 @@ describe('useReplaysFromIssue', () => {
     features: ['session-replay'],
   });
 
-  it('should fetch a list of replay ids', async () => {
+  it.isKnownFlake('should fetch a list of replay ids', async () => {
     const MOCK_GROUP = GroupFixture();
 
     MockApiClient.addMockResponse({

static/gsAdmin/views/customerDetails.spec.tsx

@@ -1257,7 +1257,7 @@ describe('Customer Details', () => {
       permissions: new Set(['billing.admin']),
     });
 
-    it('renders disabled without billing.admin permissions', async () => {
+    it.isKnownFlake('renders disabled without billing.admin permissions', async () => {
       ConfigStore.set('user', mockUser);
 
       setUpMocks(organization, {isBillingAdmin: false});

tests/js/sentry-test/isKnownFlake.d.ts

@@ -0,0 +1,12 @@
+declare namespace jest {
+  interface It {
+    /**
+     * Marks a test as a known flake. When the RERUN_KNOWN_FLAKY_TESTS env var
+     * is set (via the "Frontend: Rerun Flaky Tests" PR label), the test runs
+     * 50x to validate that a fix is stable. Otherwise it runs once, normally.
+     *
+     * Available globally — no import needed.
+     */
+    isKnownFlake(name: string, fn: jest.ProvidesCallback, timeout?: number): void;
+  }
+}