feat(test): Add it.knownFlake for stress-testing flaky Jest fixes

Adds infrastructure for validating flaky test fixes in CI:

- `itRepeatsWhenFlaky()`: a test wrapper in tests/js/sentry-test/ that
  runs a test 50x when the RERUN_KNOWN_FLAKY_TESTS env var is set,
  otherwise runs once as a normal it()
- CI wiring: frontend.yml sets RERUN_KNOWN_FLAKY_TESTS=true when the PR
  has the "Frontend: Rerun Flaky Tests" label
- ESLint: configured jest/no-standalone-expect to recognize
  itRepeatsWhenFlaky as a test block

Wraps all 13 known flaky tests (identified from 30 days of CI failures
on master) with itRepeatsWhenFlaky so fixes can be stress-tested:
- eventReplay/index.spec.tsx (6 occ)
- stackTrace.spec.tsx (5 occ)
- resultsSearchQueryBuilder.spec.tsx (5 occ, 2 tests)
- metricsTab.spec.tsx (4 occ)
- customerDetails.spec.tsx (4 occ)
- eventsSearchBar.spec.tsx (3 occ)
- trace.spec.tsx (3 occ, previously skipped)
- allMonitors.spec.tsx (2 occ)
- spansSearchBar.spec.tsx (2 occ)
- react-native/metrics.spec.tsx (2 occ)
- useReplaysFromIssue.spec.tsx (2 occ)
- spanEvidencePreview.spec.tsx (2 occ)
- groupingInfoSection.spec.tsx (2 occ)

Made-with: Cursor

Author: JoshuaKGoldberg

.github/workflows/frontend.yml

@@ -158,6 +158,9 @@ jobs:
           #
           # This quiets up the logs quite a bit.
           DEBUG_PRINT_LIMIT: 0
+          # When the "Frontend: Rerun Flaky Tests" label is on the PR,
+          # tests wrapped with it.knownFlake() run 50x to validate fixes.
+          RERUN_KNOWN_FLAKY_TESTS: "${{ contains(github.event.pull_request.labels.*.name, 'Frontend: Rerun Flaky Tests') }}"
         run: pnpm run test-ci --forceExit
 
   form-field-registry:

eslint.config.ts

@@ -826,6 +826,10 @@ export default typescript.config([
       'jest/expect-expect': 'off', // Disabled as we have many tests which render as simple validations
       'jest/no-conditional-expect': 'off', // TODO(ryan953): Fix violations then delete this line
       'jest/no-disabled-tests': 'error', // `recommended` set this to warn, we've upgraded to error
+      'jest/no-standalone-expect': [
+        'error',
+        {additionalTestBlockFunctions: ['it.knownFlake']},
+      ],
     },
   },
   {

static/app/components/events/eventReplay/index.spec.tsx

@@ -132,20 +132,23 @@ describe('EventReplay', () => {
     });
   });
 
-  it('should render the replay inline onboarding component when replays are enabled and the project supports replay', async () => {
-    MockUseReplayOnboardingSidebarPanel.mockReturnValue({
-      activateSidebar: jest.fn(),
-    });
-    MockApiClient.addMockResponse({
-      url: '/organizations/org-slug/prompts-activity/',
-      body: {data: {dismissed_ts: null}},
-    });
-    render(<EventReplay {...defaultProps} />, {organization});
-
-    expect(
-      await screen.findByText('Watch the errors and latency issues your users face')
-    ).toBeInTheDocument();
-  });
+  it.knownFlake(
+    'should render the replay inline onboarding component when replays are enabled and the project supports replay',
+    async () => {
+      MockUseReplayOnboardingSidebarPanel.mockReturnValue({
+        activateSidebar: jest.fn(),
+      });
+      MockApiClient.addMockResponse({
+        url: '/organizations/org-slug/prompts-activity/',
+        body: {data: {dismissed_ts: null}},
+      });
+      render(<EventReplay {...defaultProps} />, {organization});
+
+      expect(
+        await screen.findByText('Watch the errors and latency issues your users face')
+      ).toBeInTheDocument();
+    }
+  );
 
   it('should render a replay when there is a replayId from tags', async () => {
     MockUseReplayOnboardingSidebarPanel.mockReturnValue({

static/app/components/events/groupingInfo/groupingInfoSection.spec.tsx

@@ -45,7 +45,7 @@ describe('EventGroupingInfo', () => {
     });
   });
 
-  it('fetches and renders grouping info for errors', async () => {
+  it.knownFlake('fetches and renders grouping info for errors', async () => {
     render(<EventGroupingInfoSection {...defaultProps} />);
     await userEvent.click(
       screen.getByRole('button', {name: 'View Event Grouping Information Section'})

static/app/components/groupPreviewTooltip/spanEvidencePreview.spec.tsx

@@ -28,7 +28,7 @@ describe('SpanEvidencePreview', () => {
     expect(mock).not.toHaveBeenCalled();
   });
 
-  it('shows error when request fails', async () => {
+  it.knownFlake('shows error when request fails', async () => {
     MockApiClient.addMockResponse({
       url: `/organizations/org-slug/issues/group-id/events/recommended/`,
       body: {},

static/app/components/stackTrace/stackTrace.spec.tsx

@@ -877,7 +877,7 @@ describe('Core StackTrace', () => {
     ).toBeInTheDocument();
   });
 
-  it('shows URL link in tooltip when absPath is an http URL', async () => {
+  it.knownFlake('shows URL link in tooltip when absPath is an http URL', async () => {
     jest.useFakeTimers();
     const {event, stacktrace} = makeStackTraceData();
     const frame = stacktrace.frames[stacktrace.frames.length - 1]!;

static/app/gettingStartedDocs/react-native/metrics.spec.tsx

@@ -28,7 +28,7 @@ function renderMockRequests({
 }
 
 describe('getting started with react-native', () => {
-  it('shows React Native metrics onboarding content', async () => {
+  it.knownFlake('shows React Native metrics onboarding content', async () => {
     const organization = OrganizationFixture();
     const project = ProjectFixture({platform: 'react-native'});
     renderMockRequests({organization, project});

static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/eventsSearchBar.spec.tsx

@@ -27,7 +27,7 @@ describe('EventsSearchBar', () => {
     });
   });
 
-  it('does not show function tags in has: dropdown', async () => {
+  it.knownFlake('does not show function tags in has: dropdown', async () => {
     render(
       <EventsSearchBar
         onClose={jest.fn()}

static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/spansSearchBar.spec.tsx

@@ -122,7 +122,7 @@ describe('SpansSearchBar', () => {
     await screen.findByLabelText('span.op:function');
   });
 
-  it('calls onSearch with the correct query', async () => {
+  it.knownFlake('calls onSearch with the correct query', async () => {
     const onSearch = jest.fn();
 
     renderWithProvider({

static/app/views/detectors/list/allMonitors.spec.tsx

@@ -483,81 +483,84 @@ describe('DetectorsList', () => {
       expect(screen.getByRole('button', {name: 'Delete'})).toBeDisabled();
     });
 
-    it('shows option to select all query results when page is selected', async () => {
-      const deleteRequest = MockApiClient.addMockResponse({
-        url: '/organizations/org-slug/detectors/',
-        method: 'DELETE',
-        body: {},
-      });
-
-      render(<AllMonitors />, {organization});
-      renderGlobalModal();
-
-      const testUser = UserFixture({id: '2', email: 'test@example.com'});
-      // Mock the filtered search results - this will be used when search is applied
-      const filteredDetectors = Array.from({length: 20}, (_, i) =>
-        MetricDetectorFixture({
-          id: `filtered-${i}`,
-          name: `Assigned Detector ${i + 1}`,
-          owner: ActorFixture({id: testUser.id, name: testUser.email, type: 'user'}),
-        })
-      );
-
-      MockApiClient.addMockResponse({
-        url: '/organizations/org-slug/detectors/',
-        body: filteredDetectors,
-        headers: {
-          'X-Hits': '50',
-        },
-        match: [
-          MockApiClient.matchQuery({
-            query: '!type:issue_stream assignee:test@example.com',
-          }),
-        ],
-      });
-
-      // Click through menus to select assignee
-      const searchInput = await screen.findByRole('combobox', {
-        name: 'Add a search term',
-      });
-      await userEvent.type(searchInput, 'assignee:test@example.com{enter}');
-
-      // Wait for filtered results to load
-      await screen.findByText('Assigned Detector 1');
-
-      const rows = screen.getAllByTestId('detector-list-row');
-
-      // Focus on first row to make checkbox visible
-      await userEvent.click(rows[0]!);
-      const firstRowCheckbox = within(rows[0]!).getByRole('checkbox');
-      await userEvent.click(firstRowCheckbox);
-      expect(firstRowCheckbox).toBeChecked();
-
-      // Select all on page - master checkbox should now be visible since we have a selection
-      const masterCheckbox = screen.getAllByRole('checkbox')[0]!;
-      await userEvent.click(masterCheckbox);
-
-      // Should show alert with option to select all query results
-      expect(screen.getByText(/20 monitors on this page selected/)).toBeInTheDocument();
-      const selectAllForQuery = screen.getByRole('button', {
-        name: /Select all 50 monitors that match this search query/,
-      });
-      await userEvent.click(selectAllForQuery);
-
-      // Perform an action to verify query-based selection
-      await userEvent.click(screen.getByRole('button', {name: 'Delete'}));
-      const confirmModal = await screen.findByRole('dialog');
-      await userEvent.click(within(confirmModal).getByRole('button', {name: 'Delete'})); // Confirm
-
-      await waitFor(() => {
-        expect(deleteRequest).toHaveBeenCalledWith(
-          '/organizations/org-slug/detectors/',
-          expect.objectContaining({
-            query: {id: undefined, query: 'assignee:test@example.com', project: [1]},
+    it.knownFlake(
+      'shows option to select all query results when page is selected',
+      async () => {
+        const deleteRequest = MockApiClient.addMockResponse({
+          url: '/organizations/org-slug/detectors/',
+          method: 'DELETE',
+          body: {},
+        });
+
+        render(<AllMonitors />, {organization});
+        renderGlobalModal();
+
+        const testUser = UserFixture({id: '2', email: 'test@example.com'});
+        // Mock the filtered search results - this will be used when search is applied
+        const filteredDetectors = Array.from({length: 20}, (_, i) =>
+          MetricDetectorFixture({
+            id: `filtered-${i}`,
+            name: `Assigned Detector ${i + 1}`,
+            owner: ActorFixture({id: testUser.id, name: testUser.email, type: 'user'}),
           })
         );
-      });
-    });
+
+        MockApiClient.addMockResponse({
+          url: '/organizations/org-slug/detectors/',
+          body: filteredDetectors,
+          headers: {
+            'X-Hits': '50',
+          },
+          match: [
+            MockApiClient.matchQuery({
+              query: '!type:issue_stream assignee:test@example.com',
+            }),
+          ],
+        });
+
+        // Click through menus to select assignee
+        const searchInput = await screen.findByRole('combobox', {
+          name: 'Add a search term',
+        });
+        await userEvent.type(searchInput, 'assignee:test@example.com{enter}');
+
+        // Wait for filtered results to load
+        await screen.findByText('Assigned Detector 1');
+
+        const rows = screen.getAllByTestId('detector-list-row');
+
+        // Focus on first row to make checkbox visible
+        await userEvent.click(rows[0]!);
+        const firstRowCheckbox = within(rows[0]!).getByRole('checkbox');
+        await userEvent.click(firstRowCheckbox);
+        expect(firstRowCheckbox).toBeChecked();
+
+        // Select all on page - master checkbox should now be visible since we have a selection
+        const masterCheckbox = screen.getAllByRole('checkbox')[0]!;
+        await userEvent.click(masterCheckbox);
+
+        // Should show alert with option to select all query results
+        expect(screen.getByText(/20 monitors on this page selected/)).toBeInTheDocument();
+        const selectAllForQuery = screen.getByRole('button', {
+          name: /Select all 50 monitors that match this search query/,
+        });
+        await userEvent.click(selectAllForQuery);
+
+        // Perform an action to verify query-based selection
+        await userEvent.click(screen.getByRole('button', {name: 'Delete'}));
+        const confirmModal = await screen.findByRole('dialog');
+        await userEvent.click(within(confirmModal).getByRole('button', {name: 'Delete'})); // Confirm
+
+        await waitFor(() => {
+          expect(deleteRequest).toHaveBeenCalledWith(
+            '/organizations/org-slug/detectors/',
+            expect.objectContaining({
+              query: {id: undefined, query: 'assignee:test@example.com', project: [1]},
+            })
+          );
+        });
+      }
+    );
 
     it('disables action buttons when user does not have permissions', async () => {
       const noPermsOrganization = OrganizationFixture({