feat(test): add it.isKnownFlake for opt-in stress-testing flaky Jest fixes (#111860)

JoshuaKGoldberg · getsantry[bot] · claude · web-flow · commit df89359d6b97 · 2026-04-07T14:41:17.000-04:00
It's hard to determine when a known flaky Jest test is no longer flaky. You basically just have to keep running it repeatedly. But it takes a _long_ time (and bogs up our GHA workflows) to re-run all tests too. This PR adds plumbing that lets us specifically re-run known flaky tests 50x in a single run: 1. I added an opt-in https://github.com/getsentry/sentry/labels/Frontend%3A%20Rerun%20Flaky%20Tests label as seen on this PR 2. When that label is present, the frontend workflow sets a `RERUN_KNOWN_FLAKY_TESTS` process env var 3. Tests defined with the new `it.isKnownFlake` will define the same test 50x with incremented counter names when that var is present [Example CI failure (search `flaky rerun`)](https://github.com/getsentry/sentry/actions/runs/23800263946/job/69358610839?pr=111860): ```plaintext ● EventGroupingInfo › [flaky rerun x50] fetches and renders grouping info for errors › run 2/50 ``` I added it to the following tests that have failed >=2x on `master` over the last month: | File | Test | CI failures (30d) | Ticket | |------|------|-------------------|--------| | `eventReplay/index.spec.tsx` | render replay inline onboarding | 6 | [REPLAY-879](https://linear.app/getsentry/issue/REPLAY-879) | | `stackTrace.spec.tsx` | URL link in tooltip | 5 | [ENG-7192](https://linear.app/getsentry/issue/ENG-7192) | | `resultsSearchQueryBuilder.spec.tsx` | has: dropdown + normal tags (2 tests) | 5 | [ENG-7201](https://linear.app/getsentry/issue/ENG-7201) | | `metricsTab.spec.tsx` | toggle query builder sidebar | 4 | [ENG-7202](https://linear.app/getsentry/issue/ENG-7202) | | `customerDetails.spec.tsx` | disabled without billing.admin | 4 | [ENG-7203](https://linear.app/getsentry/issue/ENG-7203) | | `eventsSearchBar.spec.tsx` | has: dropdown | 3 | [DAIN-1271](https://linear.app/getsentry/issue/DAIN-1271) | | `trace.spec.tsx` | arrowup+shift scroll (was `it.skip`) | 3 | [BROWSE-411](https://linear.app/getsentry/issue/BROWSE-411) | | `allMonitors.spec.tsx` | select all query results | 2 | [ENG-7204](https://linear.app/getsentry/issue/ENG-7204) | | `spansSearchBar.spec.tsx` | onSearch correct query | 2 | [ENG-7205](https://linear.app/getsentry/issue/ENG-7205) | | `react-native/metrics.spec.tsx` | onboarding content | 2 | [ENG-7206](https://linear.app/getsentry/issue/ENG-7206) | | `useReplaysFromIssue.spec.tsx` | fetch replay ids | 2 | [ENG-7207](https://linear.app/getsentry/issue/ENG-7207) | | `spanEvidencePreview.spec.tsx` | error on request fail | 2 | [ENG-7208](https://linear.app/getsentry/issue/ENG-7208) | | `groupingInfoSection.spec.tsx` | render grouping info | 2 | [ENG-7209](https://linear.app/getsentry/issue/ENG-7209) | | `timeSince.spec.tsx` | respects timezone in tooltip | 1 | [ENG-7211](https://linear.app/getsentry/issue/ENG-7211) | | `versionHoverCard.spec.tsx` | renders | 1 | [ENG-7212](https://linear.app/getsentry/issue/ENG-7212) | Made with [Cursor](https://cursor.com) --------- Co-authored-by: getsantry[bot] <66042841+getsantry[bot]@users.noreply.github.com> Co-authored-by: Claude Sonnet 4 <noreply@anthropic.com>
diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
@@ -159,6 +159,9 @@ jobs:
           #
           # This quiets up the logs quite a bit.
           DEBUG_PRINT_LIMIT: 0
+          # When the "Frontend: Rerun Flaky Tests" label is on the PR,
+          # tests wrapped with it.isKnownFlake() run 50x to validate fixes.
+          RERUN_KNOWN_FLAKY_TESTS: "${{ contains(github.event.pull_request.labels.*.name, 'Frontend: Rerun Flaky Tests') }}"
         run: pnpm run test-ci --forceExit
 
   form-field-registry:
diff --git a/eslint.config.ts b/eslint.config.ts
@@ -824,6 +824,10 @@ export default typescript.config([
       'jest/expect-expect': 'off', // Disabled as we have many tests which render as simple validations
       'jest/no-conditional-expect': 'off', // TODO(ryan953): Fix violations then delete this line
       'jest/no-disabled-tests': 'error', // `recommended` set this to warn, we've upgraded to error
+      'jest/no-standalone-expect': [
+        'error',
+        {additionalTestBlockFunctions: ['it.isKnownFlake']},
+      ],
     },
   },
   {
diff --git a/static/app/components/events/eventReplay/index.spec.tsx b/static/app/components/events/eventReplay/index.spec.tsx
@@ -137,14 +137,17 @@ describe('EventReplay', () => {
     });
   });
 
-  it('should render the replay inline onboarding component when replays are enabled and the project supports replay', async () => {
-    MockUseReplayOnboardingSidebarPanel.mockReturnValue({
-      activateSidebar: jest.fn(),
-    });
-    render(<EventReplay {...defaultProps} />, {organization});
-
-    expect(await screen.findByTestId('replay-inline-onboarding')).toBeInTheDocument();
-  });
+  it.isKnownFlake(
+    'should render the replay inline onboarding component when replays are enabled and the project supports replay',
+    async () => {
+      MockUseReplayOnboardingSidebarPanel.mockReturnValue({
+        activateSidebar: jest.fn(),
+      });
+      render(<EventReplay {...defaultProps} />, {organization});
+
+      expect(await screen.findByTestId('replay-inline-onboarding')).toBeInTheDocument();
+    }
+  );
 
   it('should render a replay when there is a replayId from tags', async () => {
     MockUseReplayOnboardingSidebarPanel.mockReturnValue({
diff --git a/static/app/components/groupPreviewTooltip/spanEvidencePreview.spec.tsx b/static/app/components/groupPreviewTooltip/spanEvidencePreview.spec.tsx
@@ -28,7 +28,7 @@ describe('SpanEvidencePreview', () => {
     expect(mock).not.toHaveBeenCalled();
   });
 
-  it('shows error when request fails', async () => {
+  it.isKnownFlake('shows error when request fails', async () => {
     MockApiClient.addMockResponse({
       url: `/organizations/org-slug/issues/group-id/events/recommended/`,
       body: {},
diff --git a/static/app/components/stackTrace/stackTrace.spec.tsx b/static/app/components/stackTrace/stackTrace.spec.tsx
@@ -877,7 +877,7 @@ describe('Core StackTrace', () => {
     ).toBeInTheDocument();
   });
 
-  it('shows URL link in tooltip when absPath is an http URL', async () => {
+  it.isKnownFlake('shows URL link in tooltip when absPath is an http URL', async () => {
     jest.useFakeTimers();
     const {event, stacktrace} = makeStackTraceData();
     const frame = stacktrace.frames[stacktrace.frames.length - 1]!;
diff --git a/static/app/components/timeSince.spec.tsx b/static/app/components/timeSince.spec.tsx
@@ -58,7 +58,7 @@ describe('TimeSince', () => {
     expect(screen.getByText('10m atrás')).toBeInTheDocument();
   });
 
-  it('respects timezone in tooltip', async () => {
+  it.isKnownFlake('respects timezone in tooltip', async () => {
     const date = new Date('2024-01-15T12:00:00Z');
     render(
       <TimezoneProvider timezone="America/New_York">
diff --git a/static/app/components/versionHoverCard.spec.tsx b/static/app/components/versionHoverCard.spec.tsx
@@ -31,7 +31,7 @@ describe('VersionHoverCard', () => {
     });
   });
 
-  it('renders', async () => {
+  it.isKnownFlake('renders', async () => {
     render(
       <VersionHoverCard
         organization={organization}
diff --git a/static/app/gettingStartedDocs/react-native/metrics.spec.tsx b/static/app/gettingStartedDocs/react-native/metrics.spec.tsx
@@ -28,7 +28,7 @@ function renderMockRequests({
 }
 
 describe('getting started with react-native', () => {
-  it('shows React Native metrics onboarding content', async () => {
+  it.isKnownFlake('shows React Native metrics onboarding content', async () => {
     const organization = OrganizationFixture();
     const project = ProjectFixture({platform: 'react-native'});
     renderMockRequests({organization, project});
diff --git a/static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/eventsSearchBar.spec.tsx b/static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/eventsSearchBar.spec.tsx
@@ -27,7 +27,7 @@ describe('EventsSearchBar', () => {
     });
   });
 
-  it('does not show function tags in has: dropdown', async () => {
+  it.isKnownFlake('does not show function tags in has: dropdown', async () => {
     render(
       <EventsSearchBar
         onClose={jest.fn()}
diff --git a/static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/spansSearchBar.spec.tsx b/static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/spansSearchBar.spec.tsx
@@ -122,7 +122,7 @@ describe('SpansSearchBar', () => {
     await screen.findByLabelText('span.op:function');
   });
 
-  it('calls onSearch with the correct query', async () => {
+  it.isKnownFlake('calls onSearch with the correct query', async () => {
     const onSearch = jest.fn();
 
     renderWithProvider({
diff --git a/static/app/views/detectors/list/allMonitors.spec.tsx b/static/app/views/detectors/list/allMonitors.spec.tsx
@@ -483,81 +483,84 @@ describe('DetectorsList', () => {
       expect(screen.getByRole('button', {name: 'Delete'})).toBeDisabled();
     });
 
-    it('shows option to select all query results when page is selected', async () => {
-      const deleteRequest = MockApiClient.addMockResponse({
-        url: '/organizations/org-slug/detectors/',
-        method: 'DELETE',
-        body: {},
-      });
-
-      render(<AllMonitors />, {organization});
-      renderGlobalModal();
-
-      const testUser = UserFixture({id: '2', email: 'test@example.com'});
-      // Mock the filtered search results - this will be used when search is applied
-      const filteredDetectors = Array.from({length: 20}, (_, i) =>
-        MetricDetectorFixture({
-          id: `filtered-${i}`,
-          name: `Assigned Detector ${i + 1}`,
-          owner: ActorFixture({id: testUser.id, name: testUser.email, type: 'user'}),
-        })
-      );
-
-      MockApiClient.addMockResponse({
-        url: '/organizations/org-slug/detectors/',
-        body: filteredDetectors,
-        headers: {
-          'X-Hits': '50',
-        },
-        match: [
-          MockApiClient.matchQuery({
-            query: '!type:issue_stream assignee:test@example.com',
-          }),
-        ],
-      });
-
-      // Click through menus to select assignee
-      const searchInput = await screen.findByRole('combobox', {
-        name: 'Add a search term',
-      });
-      await userEvent.type(searchInput, 'assignee:test@example.com{enter}');
-
-      // Wait for filtered results to load
-      await screen.findByText('Assigned Detector 1');
-
-      const rows = screen.getAllByTestId('detector-list-row');
-
-      // Focus on first row to make checkbox visible
-      await userEvent.click(rows[0]!);
-      const firstRowCheckbox = within(rows[0]!).getByRole('checkbox');
-      await userEvent.click(firstRowCheckbox);
-      expect(firstRowCheckbox).toBeChecked();
-
-      // Select all on page - master checkbox should now be visible since we have a selection
-      const masterCheckbox = screen.getAllByRole('checkbox')[0]!;
-      await userEvent.click(masterCheckbox);
-
-      // Should show alert with option to select all query results
-      expect(screen.getByText(/20 monitors on this page selected/)).toBeInTheDocument();
-      const selectAllForQuery = screen.getByRole('button', {
-        name: /Select all 50 monitors that match this search query/,
-      });
-      await userEvent.click(selectAllForQuery);
-
-      // Perform an action to verify query-based selection
-      await userEvent.click(screen.getByRole('button', {name: 'Delete'}));
-      const confirmModal = await screen.findByRole('dialog');
-      await userEvent.click(within(confirmModal).getByRole('button', {name: 'Delete'})); // Confirm
-
-      await waitFor(() => {
-        expect(deleteRequest).toHaveBeenCalledWith(
-          '/organizations/org-slug/detectors/',
-          expect.objectContaining({
-            query: {id: undefined, query: 'assignee:test@example.com', project: [1]},
+    it.isKnownFlake(
+      'shows option to select all query results when page is selected',
+      async () => {
+        const deleteRequest = MockApiClient.addMockResponse({
+          url: '/organizations/org-slug/detectors/',
+          method: 'DELETE',
+          body: {},
+        });
+
+        render(<AllMonitors />, {organization});
+        renderGlobalModal();
+
+        const testUser = UserFixture({id: '2', email: 'test@example.com'});
+        // Mock the filtered search results - this will be used when search is applied
+        const filteredDetectors = Array.from({length: 20}, (_, i) =>
+          MetricDetectorFixture({
+            id: `filtered-${i}`,
+            name: `Assigned Detector ${i + 1}`,
+            owner: ActorFixture({id: testUser.id, name: testUser.email, type: 'user'}),
           })
         );
-      });
-    });
+
+        MockApiClient.addMockResponse({
+          url: '/organizations/org-slug/detectors/',
+          body: filteredDetectors,
+          headers: {
+            'X-Hits': '50',
+          },
+          match: [
+            MockApiClient.matchQuery({
+              query: '!type:issue_stream assignee:test@example.com',
+            }),
+          ],
+        });
+
+        // Click through menus to select assignee
+        const searchInput = await screen.findByRole('combobox', {
+          name: 'Add a search term',
+        });
+        await userEvent.type(searchInput, 'assignee:test@example.com{enter}');
+
+        // Wait for filtered results to load
+        await screen.findByText('Assigned Detector 1');
+
+        const rows = screen.getAllByTestId('detector-list-row');
+
+        // Focus on first row to make checkbox visible
+        await userEvent.click(rows[0]!);
+        const firstRowCheckbox = within(rows[0]!).getByRole('checkbox');
+        await userEvent.click(firstRowCheckbox);
+        expect(firstRowCheckbox).toBeChecked();
+
+        // Select all on page - master checkbox should now be visible since we have a selection
+        const masterCheckbox = screen.getAllByRole('checkbox')[0]!;
+        await userEvent.click(masterCheckbox);
+
+        // Should show alert with option to select all query results
+        expect(screen.getByText(/20 monitors on this page selected/)).toBeInTheDocument();
+        const selectAllForQuery = screen.getByRole('button', {
+          name: /Select all 50 monitors that match this search query/,
+        });
+        await userEvent.click(selectAllForQuery);
+
+        // Perform an action to verify query-based selection
+        await userEvent.click(screen.getByRole('button', {name: 'Delete'}));
+        const confirmModal = await screen.findByRole('dialog');
+        await userEvent.click(within(confirmModal).getByRole('button', {name: 'Delete'})); // Confirm
+
+        await waitFor(() => {
+          expect(deleteRequest).toHaveBeenCalledWith(
+            '/organizations/org-slug/detectors/',
+            expect.objectContaining({
+              query: {id: undefined, query: 'assignee:test@example.com', project: [1]},
+            })
+          );
+        });
+      }
+    );
 
     it('disables action buttons when user does not have permissions', async () => {
       const noPermsOrganization = OrganizationFixture({
diff --git a/static/app/views/discover/results/resultsSearchQueryBuilder.spec.tsx b/static/app/views/discover/results/resultsSearchQueryBuilder.spec.tsx
@@ -27,7 +27,7 @@ describe('ResultsSearchQueryBuilder', () => {
     });
   });
 
-  it('does not show function tags in has: dropdown', async () => {
+  it.isKnownFlake('does not show function tags in has: dropdown', async () => {
     render(
       <ResultsSearchQueryBuilder
         query=""
@@ -59,7 +59,7 @@ describe('ResultsSearchQueryBuilder', () => {
     ).not.toBeInTheDocument();
   });
 
-  it('shows normal tags, e.g. transaction, in the dropdown', async () => {
+  it.isKnownFlake('shows normal tags, e.g. transaction, in the dropdown', async () => {
     render(
       <ResultsSearchQueryBuilder
         query=""
diff --git a/static/app/views/explore/metrics/metricsTab.spec.tsx b/static/app/views/explore/metrics/metricsTab.spec.tsx
@@ -700,36 +700,39 @@ describe('MetricsTabContent (tracemetrics-ui-refresh)', () => {
     });
   });
 
-  it('toggles the query builder sidebar with the expand control', async () => {
-    render(
-      <ProviderWrapper>
-        <MetricsTabContent datePageFilterProps={datePageFilterProps} />
-      </ProviderWrapper>,
-      {
-        initialRouterConfig,
-        organization,
-      }
-    );
+  it.isKnownFlake(
+    'toggles the query builder sidebar with the expand control',
+    async () => {
+      render(
+        <ProviderWrapper>
+          <MetricsTabContent datePageFilterProps={datePageFilterProps} />
+        </ProviderWrapper>,
+        {
+          initialRouterConfig,
+          organization,
+        }
+      );
 
-    await waitFor(() => {
-      expect(
-        within(screen.getAllByTestId('metric-toolbar')[0]!).getByRole('button', {
-          name: 'bar',
-        })
-      ).toBeInTheDocument();
-    });
+      await waitFor(() => {
+        expect(
+          within(screen.getAllByTestId('metric-toolbar')[0]!).getByRole('button', {
+            name: 'bar',
+          })
+        ).toBeInTheDocument();
+      });
 
-    expect(screen.getByRole('button', {name: 'Collapse sidebar'})).toBeInTheDocument();
+      expect(screen.getByRole('button', {name: 'Collapse sidebar'})).toBeInTheDocument();
 
-    await userEvent.click(screen.getByRole('button', {name: 'Collapse sidebar'}));
+      await userEvent.click(screen.getByRole('button', {name: 'Collapse sidebar'}));
 
-    expect(screen.queryByTestId('metric-toolbar')).not.toBeInTheDocument();
-    expect(screen.getByRole('button', {name: 'Expand sidebar'})).toBeInTheDocument();
+      expect(screen.queryByTestId('metric-toolbar')).not.toBeInTheDocument();
+      expect(screen.getByRole('button', {name: 'Expand sidebar'})).toBeInTheDocument();
 
-    await userEvent.click(screen.getByRole('button', {name: 'Expand sidebar'}));
+      await userEvent.click(screen.getByRole('button', {name: 'Expand sidebar'}));
 
-    await waitFor(() => {
-      expect(screen.getAllByTestId('metric-toolbar')).toHaveLength(1);
-    });
-  });
+      await waitFor(() => {
+        expect(screen.getAllByTestId('metric-toolbar')).toHaveLength(1);
+      });
+    }
+  );
 });
diff --git a/static/app/views/issueDetails/groupReplays/useReplaysFromIssue.spec.tsx b/static/app/views/issueDetails/groupReplays/useReplaysFromIssue.spec.tsx
@@ -21,7 +21,7 @@ describe('useReplaysFromIssue', () => {
     features: ['session-replay'],
   });
 
-  it('should fetch a list of replay ids', async () => {
+  it.isKnownFlake('should fetch a list of replay ids', async () => {
     const MOCK_GROUP = GroupFixture();
 
     MockApiClient.addMockResponse({
diff --git a/static/gsAdmin/views/customerDetails.spec.tsx b/static/gsAdmin/views/customerDetails.spec.tsx
@@ -1257,7 +1257,7 @@ describe('Customer Details', () => {
       permissions: new Set(['billing.admin']),
     });
 
-    it('renders disabled without billing.admin permissions', async () => {
+    it.isKnownFlake('renders disabled without billing.admin permissions', async () => {
       ConfigStore.set('user', mockUser);
 
       setUpMocks(organization, {isBillingAdmin: false});
diff --git a/tests/js/sentry-test/isKnownFlake.d.ts b/tests/js/sentry-test/isKnownFlake.d.ts
@@ -0,0 +1,12 @@
+declare namespace jest {
+  interface It {
+    /**
+     * Marks a test as a known flake. When the RERUN_KNOWN_FLAKY_TESTS env var
+     * is set (via the "Frontend: Rerun Flaky Tests" PR label), the test runs
+     * 50x to validate that a fix is stable. Otherwise it runs once, normally.
+     *
+     * Available globally — no import needed.
+     */
+    isKnownFlake(name: string, fn: jest.ProvidesCallback, timeout?: number): void;
+  }
+}
diff --git a/tests/js/setup.ts b/tests/js/setup.ts

Original file line number	Diff line number	Diff line change
`@@ -159,6 +159,9 @@ jobs:`
`159`	`159`	`#`
`160`	`160`	`# This quiets up the logs quite a bit.`
`161`	`161`	`DEBUG_PRINT_LIMIT: 0`
	`162`	`+ # When the "Frontend: Rerun Flaky Tests" label is on the PR,`
	`163`	`+ # tests wrapped with it.isKnownFlake() run 50x to validate fixes.`
	`164`	`+ RERUN_KNOWN_FLAKY_TESTS: "${{ contains(github.event.pull_request.labels.*.name, 'Frontend: Rerun Flaky Tests') }}"`
`162`	`165`	`run: pnpm run test-ci --forceExit`
`163`	`166`
`164`	`167`	`form-field-registry:`
Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@ function renderMockRequests({`
`28`	`28`	`}`
`29`	`29`
`30`	`30`	`describe('getting started with react-native', () => {`
`31`		`- it('shows React Native metrics onboarding content', async () => {`
	`31`	`+ it.isKnownFlake('shows React Native metrics onboarding content', async () => {`
`32`	`32`	`const organization = OrganizationFixture();`
`33`	`33`	`const project = ProjectFixture({platform: 'react-native'});`
`34`	`34`	`renderMockRequests({organization, project});`