feat(test): Add it.knownFlake for stress-testing flaky Jest fixes

JoshuaKGoldberg · JoshuaKGoldberg · commit ac09db139461 · 2026-03-31T09:30:25.000-04:00
Adds infrastructure for validating flaky test fixes in CI: - `itRepeatsWhenFlaky()`: a test wrapper in tests/js/sentry-test/ that runs a test 50x when the RERUN_KNOWN_FLAKY_TESTS env var is set, otherwise runs once as a normal it() - CI wiring: frontend.yml sets RERUN_KNOWN_FLAKY_TESTS=true when the PR has the "Frontend: Rerun Flaky Tests" label - ESLint: configured jest/no-standalone-expect to recognize itRepeatsWhenFlaky as a test block Wraps all 13 known flaky tests (identified from 30 days of CI failures on master) with itRepeatsWhenFlaky so fixes can be stress-tested: - eventReplay/index.spec.tsx (6 occ) - stackTrace.spec.tsx (5 occ) - resultsSearchQueryBuilder.spec.tsx (5 occ, 2 tests) - metricsTab.spec.tsx (4 occ) - customerDetails.spec.tsx (4 occ) - eventsSearchBar.spec.tsx (3 occ) - trace.spec.tsx (3 occ, previously skipped) - allMonitors.spec.tsx (2 occ) - spansSearchBar.spec.tsx (2 occ) - react-native/metrics.spec.tsx (2 occ) - useReplaysFromIssue.spec.tsx (2 occ) - spanEvidencePreview.spec.tsx (2 occ) - groupingInfoSection.spec.tsx (2 occ) Made-with: Cursor
diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
@@ -158,6 +158,9 @@ jobs:
           #
           # This quiets up the logs quite a bit.
           DEBUG_PRINT_LIMIT: 0
+          # When the "Frontend: Rerun Flaky Tests" label is on the PR,
+          # tests wrapped with it.knownFlake() run 50x to validate fixes.
+          RERUN_KNOWN_FLAKY_TESTS: "${{ contains(github.event.pull_request.labels.*.name, 'Frontend: Rerun Flaky Tests') }}"
         run: pnpm run test-ci --forceExit
 
   form-field-registry:
diff --git a/eslint.config.ts b/eslint.config.ts
@@ -826,6 +826,10 @@ export default typescript.config([
       'jest/expect-expect': 'off', // Disabled as we have many tests which render as simple validations
       'jest/no-conditional-expect': 'off', // TODO(ryan953): Fix violations then delete this line
       'jest/no-disabled-tests': 'error', // `recommended` set this to warn, we've upgraded to error
+      'jest/no-standalone-expect': [
+        'error',
+        {additionalTestBlockFunctions: ['it.knownFlake']},
+      ],
     },
   },
   {
diff --git a/package.json b/package.json
@@ -264,8 +264,8 @@
     "eslint-plugin-no-relative-import-paths": "^1.6.1",
     "eslint-plugin-react": "7.37.5",
     "eslint-plugin-react-hooks": "6.1.0",
-    "eslint-plugin-regexp": "^3.0.0",
     "eslint-plugin-react-you-might-not-need-an-effect": "0.5.3",
+    "eslint-plugin-regexp": "^3.0.0",
     "eslint-plugin-sentry": "^2.10.0",
     "eslint-plugin-testing-library": "^7.16.0",
     "eslint-plugin-typescript-sort-keys": "^3.3.0",
diff --git a/pyproject.toml b/pyproject.toml
@@ -370,32 +370,32 @@ ignore_missing_imports = true
 # - python3 -m tools.mypy_helpers.find_easiest_modules
 [[tool.mypy.overrides]]
 module = [
-    "sentry.api.endpoints.organization_releases",
-    "sentry.api.paginator",
-    "sentry.db.postgres.base",
-    "sentry.middleware.auth",
-    "sentry.middleware.ratelimit",
-    "sentry.net.http",
-    "sentry.release_health.metrics_sessions_v2",
-    "sentry.search.events.builder.errors",
-    "sentry.search.events.builder.metrics",
-    "sentry.search.events.datasets.filter_aliases",
-    "sentry.search.events.filter",
-    "sentry.search.snuba.executors",
-    "sentry.services.eventstore.models",
-    "sentry.snuba.metrics.query_builder",
-    "sentry.testutils.cases",
-    "tests.sentry.api.helpers.test_group_index",
-    "tests.sentry.issues.test_utils",
+  "sentry.api.endpoints.organization_releases",
+  "sentry.api.paginator",
+  "sentry.db.postgres.base",
+  "sentry.middleware.auth",
+  "sentry.middleware.ratelimit",
+  "sentry.net.http",
+  "sentry.release_health.metrics_sessions_v2",
+  "sentry.search.events.builder.errors",
+  "sentry.search.events.builder.metrics",
+  "sentry.search.events.datasets.filter_aliases",
+  "sentry.search.events.filter",
+  "sentry.search.snuba.executors",
+  "sentry.services.eventstore.models",
+  "sentry.snuba.metrics.query_builder",
+  "sentry.testutils.cases",
+  "tests.sentry.api.helpers.test_group_index",
+  "tests.sentry.issues.test_utils",
 ]
 disable_error_code = [
-    "arg-type",
-    "assignment",
-    "attr-defined",
-    "call-overload",
-    "misc",
-    "override",
-    "union-attr",
+  "arg-type",
+  "assignment",
+  "attr-defined",
+  "call-overload",
+  "misc",
+  "override",
+  "union-attr",
 ]
 # end: sentry modules with typing issues
 
diff --git a/static/app/components/events/eventReplay/index.spec.tsx b/static/app/components/events/eventReplay/index.spec.tsx
@@ -132,20 +132,23 @@ describe('EventReplay', () => {
     });
   });
 
-  it('should render the replay inline onboarding component when replays are enabled and the project supports replay', async () => {
-    MockUseReplayOnboardingSidebarPanel.mockReturnValue({
-      activateSidebar: jest.fn(),
-    });
-    MockApiClient.addMockResponse({
-      url: '/organizations/org-slug/prompts-activity/',
-      body: {data: {dismissed_ts: null}},
-    });
-    render(<EventReplay {...defaultProps} />, {organization});
-
-    expect(
-      await screen.findByText('Watch the errors and latency issues your users face')
-    ).toBeInTheDocument();
-  });
+  it.knownFlake(
+    'should render the replay inline onboarding component when replays are enabled and the project supports replay',
+    async () => {
+      MockUseReplayOnboardingSidebarPanel.mockReturnValue({
+        activateSidebar: jest.fn(),
+      });
+      MockApiClient.addMockResponse({
+        url: '/organizations/org-slug/prompts-activity/',
+        body: {data: {dismissed_ts: null}},
+      });
+      render(<EventReplay {...defaultProps} />, {organization});
+
+      expect(
+        await screen.findByText('Watch the errors and latency issues your users face')
+      ).toBeInTheDocument();
+    }
+  );
 
   it('should render a replay when there is a replayId from tags', async () => {
     MockUseReplayOnboardingSidebarPanel.mockReturnValue({
diff --git a/static/app/components/events/groupingInfo/groupingInfoSection.spec.tsx b/static/app/components/events/groupingInfo/groupingInfoSection.spec.tsx
@@ -45,7 +45,7 @@ describe('EventGroupingInfo', () => {
     });
   });
 
-  it('fetches and renders grouping info for errors', async () => {
+  it.knownFlake('fetches and renders grouping info for errors', async () => {
     render(<EventGroupingInfoSection {...defaultProps} />);
     await userEvent.click(
       screen.getByRole('button', {name: 'View Event Grouping Information Section'})
diff --git a/static/app/components/groupPreviewTooltip/spanEvidencePreview.spec.tsx b/static/app/components/groupPreviewTooltip/spanEvidencePreview.spec.tsx
@@ -28,7 +28,7 @@ describe('SpanEvidencePreview', () => {
     expect(mock).not.toHaveBeenCalled();
   });
 
-  it('shows error when request fails', async () => {
+  it.knownFlake('shows error when request fails', async () => {
     MockApiClient.addMockResponse({
       url: `/organizations/org-slug/issues/group-id/events/recommended/`,
       body: {},
diff --git a/static/app/components/stackTrace/stackTrace.spec.tsx b/static/app/components/stackTrace/stackTrace.spec.tsx
@@ -877,7 +877,7 @@ describe('Core StackTrace', () => {
     ).toBeInTheDocument();
   });
 
-  it('shows URL link in tooltip when absPath is an http URL', async () => {
+  it.knownFlake('shows URL link in tooltip when absPath is an http URL', async () => {
     jest.useFakeTimers();
     const {event, stacktrace} = makeStackTraceData();
     const frame = stacktrace.frames[stacktrace.frames.length - 1]!;
diff --git a/static/app/gettingStartedDocs/react-native/metrics.spec.tsx b/static/app/gettingStartedDocs/react-native/metrics.spec.tsx
@@ -28,7 +28,7 @@ function renderMockRequests({
 }
 
 describe('getting started with react-native', () => {
-  it('shows React Native metrics onboarding content', async () => {
+  it.knownFlake('shows React Native metrics onboarding content', async () => {
     const organization = OrganizationFixture();
     const project = ProjectFixture({platform: 'react-native'});
     renderMockRequests({organization, project});
diff --git a/static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/eventsSearchBar.spec.tsx b/static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/eventsSearchBar.spec.tsx
@@ -27,7 +27,7 @@ describe('EventsSearchBar', () => {
     });
   });
 
-  it('does not show function tags in has: dropdown', async () => {
+  it.knownFlake('does not show function tags in has: dropdown', async () => {
     render(
       <EventsSearchBar
         onClose={jest.fn()}
diff --git a/static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/spansSearchBar.spec.tsx b/static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/spansSearchBar.spec.tsx
@@ -122,7 +122,7 @@ describe('SpansSearchBar', () => {
     await screen.findByLabelText('span.op:function');
   });
 
-  it('calls onSearch with the correct query', async () => {
+  it.knownFlake('calls onSearch with the correct query', async () => {
     const onSearch = jest.fn();
 
     renderWithProvider({
diff --git a/static/app/views/detectors/list/allMonitors.spec.tsx b/static/app/views/detectors/list/allMonitors.spec.tsx
@@ -483,81 +483,84 @@ describe('DetectorsList', () => {
       expect(screen.getByRole('button', {name: 'Delete'})).toBeDisabled();
     });
 
-    it('shows option to select all query results when page is selected', async () => {
-      const deleteRequest = MockApiClient.addMockResponse({
-        url: '/organizations/org-slug/detectors/',
-        method: 'DELETE',
-        body: {},
-      });
-
-      render(<AllMonitors />, {organization});
-      renderGlobalModal();
-
-      const testUser = UserFixture({id: '2', email: 'test@example.com'});
-      // Mock the filtered search results - this will be used when search is applied
-      const filteredDetectors = Array.from({length: 20}, (_, i) =>
-        MetricDetectorFixture({
-          id: `filtered-${i}`,
-          name: `Assigned Detector ${i + 1}`,
-          owner: ActorFixture({id: testUser.id, name: testUser.email, type: 'user'}),
-        })
-      );
-
-      MockApiClient.addMockResponse({
-        url: '/organizations/org-slug/detectors/',
-        body: filteredDetectors,
-        headers: {
-          'X-Hits': '50',
-        },
-        match: [
-          MockApiClient.matchQuery({
-            query: '!type:issue_stream assignee:test@example.com',
-          }),
-        ],
-      });
-
-      // Click through menus to select assignee
-      const searchInput = await screen.findByRole('combobox', {
-        name: 'Add a search term',
-      });
-      await userEvent.type(searchInput, 'assignee:test@example.com{enter}');
-
-      // Wait for filtered results to load
-      await screen.findByText('Assigned Detector 1');
-
-      const rows = screen.getAllByTestId('detector-list-row');
-
-      // Focus on first row to make checkbox visible
-      await userEvent.click(rows[0]!);
-      const firstRowCheckbox = within(rows[0]!).getByRole('checkbox');
-      await userEvent.click(firstRowCheckbox);
-      expect(firstRowCheckbox).toBeChecked();
-
-      // Select all on page - master checkbox should now be visible since we have a selection
-      const masterCheckbox = screen.getAllByRole('checkbox')[0]!;
-      await userEvent.click(masterCheckbox);
-
-      // Should show alert with option to select all query results
-      expect(screen.getByText(/20 monitors on this page selected/)).toBeInTheDocument();
-      const selectAllForQuery = screen.getByRole('button', {
-        name: /Select all 50 monitors that match this search query/,
-      });
-      await userEvent.click(selectAllForQuery);
-
-      // Perform an action to verify query-based selection
-      await userEvent.click(screen.getByRole('button', {name: 'Delete'}));
-      const confirmModal = await screen.findByRole('dialog');
-      await userEvent.click(within(confirmModal).getByRole('button', {name: 'Delete'})); // Confirm
-
-      await waitFor(() => {
-        expect(deleteRequest).toHaveBeenCalledWith(
-          '/organizations/org-slug/detectors/',
-          expect.objectContaining({
-            query: {id: undefined, query: 'assignee:test@example.com', project: [1]},
+    it.knownFlake(
+      'shows option to select all query results when page is selected',
+      async () => {
+        const deleteRequest = MockApiClient.addMockResponse({
+          url: '/organizations/org-slug/detectors/',
+          method: 'DELETE',
+          body: {},
+        });
+
+        render(<AllMonitors />, {organization});
+        renderGlobalModal();
+
+        const testUser = UserFixture({id: '2', email: 'test@example.com'});
+        // Mock the filtered search results - this will be used when search is applied
+        const filteredDetectors = Array.from({length: 20}, (_, i) =>
+          MetricDetectorFixture({
+            id: `filtered-${i}`,
+            name: `Assigned Detector ${i + 1}`,
+            owner: ActorFixture({id: testUser.id, name: testUser.email, type: 'user'}),
           })
         );
-      });
-    });
+
+        MockApiClient.addMockResponse({
+          url: '/organizations/org-slug/detectors/',
+          body: filteredDetectors,
+          headers: {
+            'X-Hits': '50',
+          },
+          match: [
+            MockApiClient.matchQuery({
+              query: '!type:issue_stream assignee:test@example.com',
+            }),
+          ],
+        });
+
+        // Click through menus to select assignee
+        const searchInput = await screen.findByRole('combobox', {
+          name: 'Add a search term',
+        });
+        await userEvent.type(searchInput, 'assignee:test@example.com{enter}');
+
+        // Wait for filtered results to load
+        await screen.findByText('Assigned Detector 1');
+
+        const rows = screen.getAllByTestId('detector-list-row');
+
+        // Focus on first row to make checkbox visible
+        await userEvent.click(rows[0]!);
+        const firstRowCheckbox = within(rows[0]!).getByRole('checkbox');
+        await userEvent.click(firstRowCheckbox);
+        expect(firstRowCheckbox).toBeChecked();
+
+        // Select all on page - master checkbox should now be visible since we have a selection
+        const masterCheckbox = screen.getAllByRole('checkbox')[0]!;
+        await userEvent.click(masterCheckbox);
+
+        // Should show alert with option to select all query results
+        expect(screen.getByText(/20 monitors on this page selected/)).toBeInTheDocument();
+        const selectAllForQuery = screen.getByRole('button', {
+          name: /Select all 50 monitors that match this search query/,
+        });
+        await userEvent.click(selectAllForQuery);
+
+        // Perform an action to verify query-based selection
+        await userEvent.click(screen.getByRole('button', {name: 'Delete'}));
+        const confirmModal = await screen.findByRole('dialog');
+        await userEvent.click(within(confirmModal).getByRole('button', {name: 'Delete'})); // Confirm
+
+        await waitFor(() => {
+          expect(deleteRequest).toHaveBeenCalledWith(
+            '/organizations/org-slug/detectors/',
+            expect.objectContaining({
+              query: {id: undefined, query: 'assignee:test@example.com', project: [1]},
+            })
+          );
+        });
+      }
+    );
 
     it('disables action buttons when user does not have permissions', async () => {
       const noPermsOrganization = OrganizationFixture({
diff --git a/static/app/views/discover/results/resultsSearchQueryBuilder.spec.tsx b/static/app/views/discover/results/resultsSearchQueryBuilder.spec.tsx
@@ -27,7 +27,7 @@ describe('ResultsSearchQueryBuilder', () => {
     });
   });
 
-  it('does not show function tags in has: dropdown', async () => {
+  it.knownFlake('does not show function tags in has: dropdown', async () => {
     render(
       <ResultsSearchQueryBuilder
         query=""
@@ -59,7 +59,7 @@ describe('ResultsSearchQueryBuilder', () => {
     ).not.toBeInTheDocument();
   });
 
-  it('shows normal tags, e.g. transaction, in the dropdown', async () => {
+  it.knownFlake('shows normal tags, e.g. transaction, in the dropdown', async () => {
     render(
       <ResultsSearchQueryBuilder
         query=""
diff --git a/static/app/views/explore/metrics/metricsTab.spec.tsx b/static/app/views/explore/metrics/metricsTab.spec.tsx
@@ -700,7 +700,7 @@ describe('MetricsTabContent (tracemetrics-ui-refresh)', () => {
     });
   });
 
-  it('toggles the query builder sidebar with the expand control', async () => {
+  it.knownFlake('toggles the query builder sidebar with the expand control', async () => {
     render(
       <ProviderWrapper>
         <MetricsTabContent datePageFilterProps={datePageFilterProps} />
diff --git a/static/app/views/issueDetails/groupReplays/useReplaysFromIssue.spec.tsx b/static/app/views/issueDetails/groupReplays/useReplaysFromIssue.spec.tsx
@@ -21,7 +21,7 @@ describe('useReplaysFromIssue', () => {
     features: ['session-replay'],
   });
 
-  it('should fetch a list of replay ids', async () => {
+  it.knownFlake('should fetch a list of replay ids', async () => {
     const MOCK_GROUP = GroupFixture();
 
     MockApiClient.addMockResponse({
diff --git a/static/app/views/performance/newTraceDetails/trace.spec.tsx b/static/app/views/performance/newTraceDetails/trace.spec.tsx
@@ -1451,8 +1451,7 @@ describe('trace view', () => {
       });
     });
 
-    // eslint-disable-next-line jest/no-disabled-tests
-    it.skip('arrowup+shift scrolls to the start of the list', async () => {
+    it.knownFlake('arrowup+shift scrolls to the start of the list', async () => {
       const {virtualizedContainer} = await keyboardNavigationTestSetup();
 
       let rows = getVirtualizedRows(virtualizedContainer);
diff --git a/static/gsAdmin/views/customerDetails.spec.tsx b/static/gsAdmin/views/customerDetails.spec.tsx
@@ -1257,7 +1257,7 @@ describe('Customer Details', () => {
       permissions: new Set(['billing.admin']),
     });
 
-    it('renders disabled without billing.admin permissions', async () => {
+    it.knownFlake('renders disabled without billing.admin permissions', async () => {
       ConfigStore.set('user', mockUser);
 
       setUpMocks(organization, {isBillingAdmin: false});
diff --git a/tests/js/sentry-test/knownFlake.d.ts b/tests/js/sentry-test/knownFlake.d.ts
diff --git a/tests/js/setup.ts b/tests/js/setup.ts

Original file line number	Diff line number	Diff line change
`@@ -158,6 +158,9 @@ jobs:`
`158`	`158`	`#`
`159`	`159`	`# This quiets up the logs quite a bit.`
`160`	`160`	`DEBUG_PRINT_LIMIT: 0`
	`161`	`+ # When the "Frontend: Rerun Flaky Tests" label is on the PR,`
	`162`	`+ # tests wrapped with it.knownFlake() run 50x to validate fixes.`
	`163`	`+ RERUN_KNOWN_FLAKY_TESTS: "${{ contains(github.event.pull_request.labels.*.name, 'Frontend: Rerun Flaky Tests') }}"`
`161`	`164`	`run: pnpm run test-ci --forceExit`
`162`	`165`
`163`	`166`	`form-field-registry:`
Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@ function renderMockRequests({`
`28`	`28`	`}`
`29`	`29`
`30`	`30`	`describe('getting started with react-native', () => {`
`31`		`- it('shows React Native metrics onboarding content', async () => {`
	`31`	`+ it.knownFlake('shows React Native metrics onboarding content', async () => {`
`32`	`32`	`const organization = OrganizationFixture();`
`33`	`33`	`const project = ProjectFixture({platform: 'react-native'});`
`34`	`34`	`renderMockRequests({organization, project});`