Skip to content

Commit f42d3ea

Browse files
JoshuaKGoldbergJoshuaKGoldberg
committed
feat(test): Add itRepeatsWhenFlaky infra for stress-testing flaky Jest fixes
Adds infrastructure for validating flaky test fixes in CI: - `itRepeatsWhenFlaky()`: a test wrapper in tests/js/sentry-test/ that runs a test 50x when the RERUN_KNOWN_FLAKY_TESTS env var is set, otherwise runs once as a normal it() - CI wiring: frontend.yml sets RERUN_KNOWN_FLAKY_TESTS=true when the PR has the "Frontend: Rerun Flaky Tests" label - ESLint: configured jest/no-standalone-expect to recognize itRepeatsWhenFlaky as a test block Wraps all 13 known flaky tests (identified from 30 days of CI failures on master) with itRepeatsWhenFlaky so fixes can be stress-tested: - eventReplay/index.spec.tsx (6 occ) - stackTrace.spec.tsx (5 occ) - resultsSearchQueryBuilder.spec.tsx (5 occ, 2 tests) - metricsTab.spec.tsx (4 occ) - customerDetails.spec.tsx (4 occ) - eventsSearchBar.spec.tsx (3 occ) - trace.spec.tsx (3 occ, previously skipped) - allMonitors.spec.tsx (2 occ) - spansSearchBar.spec.tsx (2 occ) - react-native/metrics.spec.tsx (2 occ) - useReplaysFromIssue.spec.tsx (2 occ) - spanEvidencePreview.spec.tsx (2 occ) - groupingInfoSection.spec.tsx (2 occ) Made-with: Cursor
1 parent 9ba2fea commit f42d3ea

File tree

16 files changed

+213
-156
lines changed

16 files changed

+213
-156
lines changed

‎.github/workflows/frontend.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,9 @@ jobs:
158158
#
159159
# This quiets up the logs quite a bit.
160160
DEBUG_PRINT_LIMIT: 0
161+
# When the "Frontend: Rerun Flaky Tests" label is on the PR,
162+
# tests wrapped with itRepeatsWhenFlaky() run 50x to validate fixes.
163+
RERUN_KNOWN_FLAKY_TESTS: "${{ contains(github.event.pull_request.labels.*.name, 'Frontend: Rerun Flaky Tests') }}"
161164
run: pnpm run test-ci --forceExit
162165

163166
form-field-registry:

‎eslint.config.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -826,6 +826,10 @@ export default typescript.config([
826826
'jest/expect-expect': 'off', // Disabled as we have many tests which render as simple validations
827827
'jest/no-conditional-expect': 'off', // TODO(ryan953): Fix violations then delete this line
828828
'jest/no-disabled-tests': 'error', // `recommended` set this to warn, we've upgraded to error
829+
'jest/no-standalone-expect': [
830+
'error',
831+
{additionalTestBlockFunctions: ['itRepeatsWhenFlaky']},
832+
],
829833
},
830834
},
831835
{

‎static/app/components/events/eventReplay/index.spec.tsx‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {RRWebInitFrameEventsFixture} from 'sentry-fixture/replay/rrweb';
66
import {ReplayRecordFixture} from 'sentry-fixture/replayRecord';
77
import {UserFixture} from 'sentry-fixture/user';
88

9+
import {itRepeatsWhenFlaky} from 'sentry-test/flakyTestRerun';
910
import {render, screen} from 'sentry-test/reactTestingLibrary';
1011

1112
import {EventReplay} from 'sentry/components/events/eventReplay';
@@ -132,20 +133,23 @@ describe('EventReplay', () => {
132133
});
133134
});
134135

135-
it('should render the replay inline onboarding component when replays are enabled and the project supports replay', async () => {
136-
MockUseReplayOnboardingSidebarPanel.mockReturnValue({
137-
activateSidebar: jest.fn(),
138-
});
139-
MockApiClient.addMockResponse({
140-
url: '/organizations/org-slug/prompts-activity/',
141-
body: {data: {dismissed_ts: null}},
142-
});
143-
render(<EventReplay {...defaultProps} />, {organization});
144-
145-
expect(
146-
await screen.findByText('Watch the errors and latency issues your users face')
147-
).toBeInTheDocument();
148-
});
136+
itRepeatsWhenFlaky(
137+
'should render the replay inline onboarding component when replays are enabled and the project supports replay',
138+
async () => {
139+
MockUseReplayOnboardingSidebarPanel.mockReturnValue({
140+
activateSidebar: jest.fn(),
141+
});
142+
MockApiClient.addMockResponse({
143+
url: '/organizations/org-slug/prompts-activity/',
144+
body: {data: {dismissed_ts: null}},
145+
});
146+
render(<EventReplay {...defaultProps} />, {organization});
147+
148+
expect(
149+
await screen.findByText('Watch the errors and latency issues your users face')
150+
).toBeInTheDocument();
151+
}
152+
);
149153

150154
it('should render a replay when there is a replayId from tags', async () => {
151155
MockUseReplayOnboardingSidebarPanel.mockReturnValue({

‎static/app/components/events/groupingInfo/groupingInfoSection.spec.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import {EventFixture} from 'sentry-fixture/event';
22
import {GroupFixture} from 'sentry-fixture/group';
33

4+
import {itRepeatsWhenFlaky} from 'sentry-test/flakyTestRerun';
45
import {render, screen, userEvent} from 'sentry-test/reactTestingLibrary';
56

67
import {EventGroupVariantType} from 'sentry/types/event';
@@ -45,7 +46,7 @@ describe('EventGroupingInfo', () => {
4546
});
4647
});
4748

48-
it('fetches and renders grouping info for errors', async () => {
49+
itRepeatsWhenFlaky('fetches and renders grouping info for errors', async () => {
4950
render(<EventGroupingInfoSection {...defaultProps} />);
5051
await userEvent.click(
5152
screen.getByRole('button', {name: 'View Event Grouping Information Section'})

‎static/app/components/groupPreviewTooltip/spanEvidencePreview.spec.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import {itRepeatsWhenFlaky} from 'sentry-test/flakyTestRerun';
12
import {
23
MockSpan,
34
ProblemSpan,
@@ -28,7 +29,7 @@ describe('SpanEvidencePreview', () => {
2829
expect(mock).not.toHaveBeenCalled();
2930
});
3031

31-
it('shows error when request fails', async () => {
32+
itRepeatsWhenFlaky('shows error when request fails', async () => {
3233
MockApiClient.addMockResponse({
3334
url: `/organizations/org-slug/issues/group-id/events/recommended/`,
3435
body: {},

‎static/app/components/stackTrace/stackTrace.spec.tsx‎

Lines changed: 36 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {GitHubIntegrationFixture} from 'sentry-fixture/githubIntegration';
66
import {OrganizationFixture} from 'sentry-fixture/organization';
77
import {ProjectFixture} from 'sentry-fixture/project';
88

9+
import {itRepeatsWhenFlaky} from 'sentry-test/flakyTestRerun';
910
import {act, render, screen, userEvent, within} from 'sentry-test/reactTestingLibrary';
1011
import {textWithMarkupMatcher} from 'sentry-test/utils';
1112

@@ -877,36 +878,39 @@ describe('Core StackTrace', () => {
877878
).toBeInTheDocument();
878879
});
879880

880-
it('shows URL link in tooltip when absPath is an http URL', async () => {
881-
jest.useFakeTimers();
882-
const {event, stacktrace} = makeStackTraceData();
883-
const frame = stacktrace.frames[stacktrace.frames.length - 1]!;
884-
885-
render(
886-
<TestStackTraceProvider
887-
event={event}
888-
stacktrace={{
889-
...stacktrace,
890-
frames: [
891-
{
892-
...frame,
893-
absPath: 'https://example.com/static/app.js',
894-
filename: 'app.js',
895-
inApp: true,
896-
},
897-
],
898-
}}
899-
>
900-
<StackTraceFrames frameContextComponent={FrameContent} />
901-
</TestStackTraceProvider>
902-
);
903-
904-
await userEvent.hover(screen.getByText('app.js'), {delay: null});
905-
act(() => jest.advanceTimersByTime(2000));
906-
907-
expect(
908-
await screen.findByRole('link', {name: 'https://example.com/static/app.js'})
909-
).toBeInTheDocument();
910-
jest.useRealTimers();
911-
});
881+
itRepeatsWhenFlaky(
882+
'shows URL link in tooltip when absPath is an http URL',
883+
async () => {
884+
jest.useFakeTimers();
885+
const {event, stacktrace} = makeStackTraceData();
886+
const frame = stacktrace.frames[stacktrace.frames.length - 1]!;
887+
888+
render(
889+
<TestStackTraceProvider
890+
event={event}
891+
stacktrace={{
892+
...stacktrace,
893+
frames: [
894+
{
895+
...frame,
896+
absPath: 'https://example.com/static/app.js',
897+
filename: 'app.js',
898+
inApp: true,
899+
},
900+
],
901+
}}
902+
>
903+
<StackTraceFrames frameContextComponent={FrameContent} />
904+
</TestStackTraceProvider>
905+
);
906+
907+
await userEvent.hover(screen.getByText('app.js'), {delay: null});
908+
act(() => jest.advanceTimersByTime(2000));
909+
910+
expect(
911+
await screen.findByRole('link', {name: 'https://example.com/static/app.js'})
912+
).toBeInTheDocument();
913+
jest.useRealTimers();
914+
}
915+
);
912916
});

‎static/app/gettingStartedDocs/react-native/metrics.spec.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import {OrganizationFixture} from 'sentry-fixture/organization';
22
import {ProjectFixture} from 'sentry-fixture/project';
33
import {ProjectKeysFixture} from 'sentry-fixture/projectKeys';
44

5+
import {itRepeatsWhenFlaky} from 'sentry-test/flakyTestRerun';
56
import {render, screen, userEvent} from 'sentry-test/reactTestingLibrary';
67

78
import type {Organization} from 'sentry/types/organization';
@@ -28,7 +29,7 @@ function renderMockRequests({
2829
}
2930

3031
describe('getting started with react-native', () => {
31-
it('shows React Native metrics onboarding content', async () => {
32+
itRepeatsWhenFlaky('shows React Native metrics onboarding content', async () => {
3233
const organization = OrganizationFixture();
3334
const project = ProjectFixture({platform: 'react-native'});
3435
renderMockRequests({organization, project});

‎static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/eventsSearchBar.spec.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import {OrganizationFixture} from 'sentry-fixture/organization';
22
import {PageFiltersFixture} from 'sentry-fixture/pageFilters';
33

4+
import {itRepeatsWhenFlaky} from 'sentry-test/flakyTestRerun';
45
import {render, screen, userEvent, within} from 'sentry-test/reactTestingLibrary';
56

67
import type {Organization} from 'sentry/types/organization';
@@ -27,7 +28,7 @@ describe('EventsSearchBar', () => {
2728
});
2829
});
2930

30-
it('does not show function tags in has: dropdown', async () => {
31+
itRepeatsWhenFlaky('does not show function tags in has: dropdown', async () => {
3132
render(
3233
<EventsSearchBar
3334
onClose={jest.fn()}

‎static/app/views/dashboards/widgetBuilder/buildSteps/filterResultsStep/spansSearchBar.spec.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import type {ComponentProps} from 'react';
22
import {WidgetQueryFixture} from 'sentry-fixture/widgetQuery';
33

4+
import {itRepeatsWhenFlaky} from 'sentry-test/flakyTestRerun';
45
import {render, screen, userEvent, waitFor} from 'sentry-test/reactTestingLibrary';
56

67
import {WildcardOperators} from 'sentry/components/searchSyntax/parser';
@@ -122,7 +123,7 @@ describe('SpansSearchBar', () => {
122123
await screen.findByLabelText('span.op:function');
123124
});
124125

125-
it('calls onSearch with the correct query', async () => {
126+
itRepeatsWhenFlaky('calls onSearch with the correct query', async () => {
126127
const onSearch = jest.fn();
127128

128129
renderWithProvider({

‎static/app/views/detectors/list/allMonitors.spec.tsx‎

Lines changed: 77 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {OrganizationFixture} from 'sentry-fixture/organization';
55
import {PageFiltersFixture} from 'sentry-fixture/pageFilters';
66
import {UserFixture} from 'sentry-fixture/user';
77

8+
import {itRepeatsWhenFlaky} from 'sentry-test/flakyTestRerun';
89
import {
910
render,
1011
renderGlobalModal,
@@ -483,81 +484,84 @@ describe('DetectorsList', () => {
483484
expect(screen.getByRole('button', {name: 'Delete'})).toBeDisabled();
484485
});
485486

486-
it('shows option to select all query results when page is selected', async () => {
487-
const deleteRequest = MockApiClient.addMockResponse({
488-
url: '/organizations/org-slug/detectors/',
489-
method: 'DELETE',
490-
body: {},
491-
});
492-
493-
render(<AllMonitors />, {organization});
494-
renderGlobalModal();
495-
496-
const testUser = UserFixture({id: '2', email: 'test@example.com'});
497-
// Mock the filtered search results - this will be used when search is applied
498-
const filteredDetectors = Array.from({length: 20}, (_, i) =>
499-
MetricDetectorFixture({
500-
id: `filtered-${i}`,
501-
name: `Assigned Detector ${i + 1}`,
502-
owner: ActorFixture({id: testUser.id, name: testUser.email, type: 'user'}),
503-
})
504-
);
505-
506-
MockApiClient.addMockResponse({
507-
url: '/organizations/org-slug/detectors/',
508-
body: filteredDetectors,
509-
headers: {
510-
'X-Hits': '50',
511-
},
512-
match: [
513-
MockApiClient.matchQuery({
514-
query: '!type:issue_stream assignee:test@example.com',
515-
}),
516-
],
517-
});
518-
519-
// Click through menus to select assignee
520-
const searchInput = await screen.findByRole('combobox', {
521-
name: 'Add a search term',
522-
});
523-
await userEvent.type(searchInput, 'assignee:test@example.com{enter}');
524-
525-
// Wait for filtered results to load
526-
await screen.findByText('Assigned Detector 1');
527-
528-
const rows = screen.getAllByTestId('detector-list-row');
529-
530-
// Focus on first row to make checkbox visible
531-
await userEvent.click(rows[0]!);
532-
const firstRowCheckbox = within(rows[0]!).getByRole('checkbox');
533-
await userEvent.click(firstRowCheckbox);
534-
expect(firstRowCheckbox).toBeChecked();
535-
536-
// Select all on page - master checkbox should now be visible since we have a selection
537-
const masterCheckbox = screen.getAllByRole('checkbox')[0]!;
538-
await userEvent.click(masterCheckbox);
539-
540-
// Should show alert with option to select all query results
541-
expect(screen.getByText(/20 monitors on this page selected/)).toBeInTheDocument();
542-
const selectAllForQuery = screen.getByRole('button', {
543-
name: /Select all 50 monitors that match this search query/,
544-
});
545-
await userEvent.click(selectAllForQuery);
546-
547-
// Perform an action to verify query-based selection
548-
await userEvent.click(screen.getByRole('button', {name: 'Delete'}));
549-
const confirmModal = await screen.findByRole('dialog');
550-
await userEvent.click(within(confirmModal).getByRole('button', {name: 'Delete'})); // Confirm
551-
552-
await waitFor(() => {
553-
expect(deleteRequest).toHaveBeenCalledWith(
554-
'/organizations/org-slug/detectors/',
555-
expect.objectContaining({
556-
query: {id: undefined, query: 'assignee:test@example.com', project: [1]},
487+
itRepeatsWhenFlaky(
488+
'shows option to select all query results when page is selected',
489+
async () => {
490+
const deleteRequest = MockApiClient.addMockResponse({
491+
url: '/organizations/org-slug/detectors/',
492+
method: 'DELETE',
493+
body: {},
494+
});
495+
496+
render(<AllMonitors />, {organization});
497+
renderGlobalModal();
498+
499+
const testUser = UserFixture({id: '2', email: 'test@example.com'});
500+
// Mock the filtered search results - this will be used when search is applied
501+
const filteredDetectors = Array.from({length: 20}, (_, i) =>
502+
MetricDetectorFixture({
503+
id: `filtered-${i}`,
504+
name: `Assigned Detector ${i + 1}`,
505+
owner: ActorFixture({id: testUser.id, name: testUser.email, type: 'user'}),
557506
})
558507
);
559-
});
560-
});
508+
509+
MockApiClient.addMockResponse({
510+
url: '/organizations/org-slug/detectors/',
511+
body: filteredDetectors,
512+
headers: {
513+
'X-Hits': '50',
514+
},
515+
match: [
516+
MockApiClient.matchQuery({
517+
query: '!type:issue_stream assignee:test@example.com',
518+
}),
519+
],
520+
});
521+
522+
// Click through menus to select assignee
523+
const searchInput = await screen.findByRole('combobox', {
524+
name: 'Add a search term',
525+
});
526+
await userEvent.type(searchInput, 'assignee:test@example.com{enter}');
527+
528+
// Wait for filtered results to load
529+
await screen.findByText('Assigned Detector 1');
530+
531+
const rows = screen.getAllByTestId('detector-list-row');
532+
533+
// Focus on first row to make checkbox visible
534+
await userEvent.click(rows[0]!);
535+
const firstRowCheckbox = within(rows[0]!).getByRole('checkbox');
536+
await userEvent.click(firstRowCheckbox);
537+
expect(firstRowCheckbox).toBeChecked();
538+
539+
// Select all on page - master checkbox should now be visible since we have a selection
540+
const masterCheckbox = screen.getAllByRole('checkbox')[0]!;
541+
await userEvent.click(masterCheckbox);
542+
543+
// Should show alert with option to select all query results
544+
expect(screen.getByText(/20 monitors on this page selected/)).toBeInTheDocument();
545+
const selectAllForQuery = screen.getByRole('button', {
546+
name: /Select all 50 monitors that match this search query/,
547+
});
548+
await userEvent.click(selectAllForQuery);
549+
550+
// Perform an action to verify query-based selection
551+
await userEvent.click(screen.getByRole('button', {name: 'Delete'}));
552+
const confirmModal = await screen.findByRole('dialog');
553+
await userEvent.click(within(confirmModal).getByRole('button', {name: 'Delete'})); // Confirm
554+
555+
await waitFor(() => {
556+
expect(deleteRequest).toHaveBeenCalledWith(
557+
'/organizations/org-slug/detectors/',
558+
expect.objectContaining({
559+
query: {id: undefined, query: 'assignee:test@example.com', project: [1]},
560+
})
561+
);
562+
});
563+
}
564+
);
561565

562566
it('disables action buttons when user does not have permissions', async () => {
563567
const noPermsOrganization = OrganizationFixture({

0 commit comments

Comments
 (0)