From 58f5c787007bd4a0814ee3789e2b4498e2528e05 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 12:03:39 -0800 Subject: [PATCH 01/42] Add backend selective testing workflow --- .../workflows/backend_selective_testing.yml | 83 ++++++++++++++ src/sentry/preprod/size_analysis/compare.py | 2 + .../testutils/pytest/selective_testing.py | 103 ++++++++++++++++++ src/sentry/testutils/pytest/sentry.py | 45 +++++++- 4 files changed, 231 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/backend_selective_testing.yml create mode 100644 src/sentry/testutils/pytest/selective_testing.py diff --git a/.github/workflows/backend_selective_testing.yml b/.github/workflows/backend_selective_testing.yml new file mode 100644 index 00000000000000..81783cee31261b --- /dev/null +++ b/.github/workflows/backend_selective_testing.yml @@ -0,0 +1,83 @@ +name: backend - selective + +on: + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + backend-test-selective: + name: backend test (selective) + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: read + id-token: write + actions: read # used for DIM metadata + strategy: + fail-fast: false + matrix: + instance: + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21] + + env: + MATRIX_INSTANCE_TOTAL: 22 + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + mode: backend-ci + + # TODO: Gcloud + - name: Download coverage database + uses: actions/download-artifact@v4 + with: + name: pycoverage-sqlite-combined-20529759656 + path: .coverage + run-id: 20529759656 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 + + - name: List all changed files + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + for file in ${ALL_CHANGED_FILES}; do + echo "$file was changed" + done + + - name: Run backend tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) + id: run_backend_tests + run: make test-python-ci + env: + SELECTIVE_TESTING_ENABLED: true + CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + COVERAGE_DB_PATH: .coverage + + - name: Inspect failure + if: failure() + run: | + if command -v devservices; then + devservices logs + fi + + # - name: Collect test data + # uses: ./.github/actions/collect-test-data + # if: ${{ !cancelled() }} + # with: + # artifact_path: .artifacts/pytest.json # TODO + # gcs_bucket: ${{ secrets.COLLECT_TEST_DATA_GCS_BUCKET }} + # gcp_project_id: ${{ secrets.COLLECT_TEST_DATA_GCP_PROJECT_ID }} + # workload_identity_provider: ${{ secrets.SENTRY_GCP_DEV_WORKLOAD_IDENTITY_POOL }} + # service_account_email: ${{ secrets.COLLECT_TEST_DATA_SERVICE_ACCOUNT_EMAIL }} + # matrix_instance_number: ${{ steps.setup.outputs.matrix-instance-number }} diff --git a/src/sentry/preprod/size_analysis/compare.py b/src/sentry/preprod/size_analysis/compare.py index 42d9b309236bdd..2996ca8c9d7cab 100644 --- a/src/sentry/preprod/size_analysis/compare.py +++ b/src/sentry/preprod/size_analysis/compare.py @@ -155,6 +155,8 @@ def compare_size_analysis( base_download_size=base_size_analysis.max_download_size, ) + # Placeholder + # Compare insights only if we're not skipping the comparison insight_diff_items = [] if not skip_diff_item_comparison: diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py new file mode 100644 index 00000000000000..472aaa10ddf7eb --- /dev/null +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import os +import sqlite3 +import sys + + +def _file_executed(bitblob: bytes) -> bool: + """ + Returns True if any line in the file was executed (bitblob has any bits set). + """ + return any(b != 0 for b in bitblob) + + +def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> set[str] | None: + """ + Query the coverage database to find which tests executed code in the given source files. + + Args: + db_path: Path to the .coverage SQLite database + source_files: List of source file paths that have changed + + Returns: + Set of test file paths (e.g., 'tests/sentry/api/test_foo.py'), + or None if the database doesn't exist or there's an error. + """ + if not os.path.exists(db_path): + return None + + try: + conn = sqlite3.connect(db_path) + cur = conn.cursor() + + test_contexts = set() + + for file_path in source_files: + # Query for test contexts that executed this file + cur.execute( + """ + SELECT c.context, lb.numbits + FROM line_bits lb + JOIN file f ON lb.file_id = f.id + JOIN context c ON lb.context_id = c.id + WHERE f.path LIKE '%' || ? + AND c.context != '' + """, + (f"/{file_path}",), + ) + + for context, bitblob in cur.fetchall(): + if _file_executed(bitblob): + test_contexts.add(context) + + conn.close() + + # Extract test file paths from contexts + # Context format: 'tests/foo/bar.py::TestClass::test_function' + test_files = set() + for context in test_contexts: + test_file = context.split("::", 1)[0] + test_files.add(test_file) + + return test_files + + except (sqlite3.Error, Exception) as e: + # Log the error but don't fail the test run + print(f"Warning: Could not query coverage database: {e}", file=sys.stderr) + return None + + +def filter_items_by_coverage(items, changed_files: list[str], coverage_db_path: str): + """ + Filter pytest items to only include tests affected by the changed files. + + Args: + items: List of pytest.Item objects to filter + changed_files: List of source files that have changed + coverage_db_path: Path to the coverage database + + Returns: + Tuple of (selected_items, discarded_items, affected_test_files) + where affected_test_files is the set of test files found in coverage data, + or None if coverage data could not be loaded. + """ + affected_test_files = get_affected_tests_from_coverage(coverage_db_path, changed_files) + + if affected_test_files is None: + # Could not load coverage data, return all items as selected + return list(items), [], None + + # Filter items to only include tests from affected files + selected_items = [] + discarded_items = [] + + for item in items: + # Extract test file path from nodeid (e.g., 'tests/foo.py::TestClass::test_func') + test_file = item.nodeid.split("::", 1)[0] + if test_file in affected_test_files: + selected_items.append(item) + else: + discarded_items.append(item) + + return selected_items, discarded_items, affected_test_files diff --git a/src/sentry/testutils/pytest/sentry.py b/src/sentry/testutils/pytest/sentry.py index 547e8dfc0630f8..b594efad402bc0 100644 --- a/src/sentry/testutils/pytest/sentry.py +++ b/src/sentry/testutils/pytest/sentry.py @@ -18,6 +18,7 @@ from sentry.runner.importer import install_plugin_apps from sentry.silo.base import SiloMode +from sentry.testutils.pytest.selective_testing import filter_items_by_coverage from sentry.testutils.region import TestEnvRegionDirectory from sentry.testutils.silo import monkey_patch_single_process_silo_mode_state from sentry.types import region @@ -399,6 +400,43 @@ def _shuffle_d(dct: dict[K, V]) -> dict[K, V]: def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: """After collection, we need to select tests based on group and group strategy""" + initial_discard = [] + + # Selective testing based on coverage data + if os.environ.get("SELECTIVE_TESTING_ENABLED"): + changed_files_str = os.environ.get("CHANGED_FILES", "") + coverage_db_path = os.environ.get("COVERAGE_DB_PATH", ".coverage.combined") + + if changed_files_str: + # Parse changed files from comma-separated string + changed_files = [f.strip() for f in changed_files_str.split(",") if f.strip()] + + config.get_terminal_writer().line( + f"Selective testing enabled for {len(changed_files)} changed file(s)" + ) + + # Filter tests using coverage data + selected_items, discarded_items, affected_test_files = filter_items_by_coverage( + items, changed_files, coverage_db_path + ) + + if affected_test_files is not None: + config.get_terminal_writer().line( + f"Found {len(affected_test_files)} affected test file(s) from coverage data" + ) + config.get_terminal_writer().line( + f"Selected {len(selected_items)}/{len(items)} tests based on coverage" + ) + + # Update items with filtered list + items[:] = selected_items + initial_discard = discarded_items + else: + config.get_terminal_writer().line( + "Warning: Could not load coverage data, running all tests" + ) + + # Existing grouping logic (unchanged) total_groups = int(os.environ.get("TOTAL_TEST_GROUPS", 1)) current_group = int(os.environ.get("TEST_GROUP", 0)) grouping_strategy = os.environ.get("TEST_GROUP_STRATEGY", "scope") @@ -431,9 +469,12 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item config.get_terminal_writer().line(f"SENTRY_SHUFFLE_TESTS_SEED: {seed}") _shuffle(items, random.Random(seed)) + # Combine discards from both selective testing and grouping + all_discarded = initial_discard + discard + # This only needs to be done if there are items to be de-selected - if len(discard) > 0: - config.hook.pytest_deselected(items=discard) + if len(all_discarded) > 0: + config.hook.pytest_deselected(items=all_discarded) def pytest_xdist_setupnodes() -> None: From e45a56614816f048d883608012688356de29d2a7 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 14:43:40 -0800 Subject: [PATCH 02/42] Tweak --- .github/workflows/backend_selective_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend_selective_testing.yml b/.github/workflows/backend_selective_testing.yml index 81783cee31261b..3dc9e3d238fc55 100644 --- a/.github/workflows/backend_selective_testing.yml +++ b/.github/workflows/backend_selective_testing.yml @@ -40,8 +40,8 @@ jobs: - name: Download coverage database uses: actions/download-artifact@v4 with: - name: pycoverage-sqlite-combined-20529759656 path: .coverage + artifact-id: 4972973279 run-id: 20529759656 - name: Get changed files From 7f8652b80482ccca3c3b6b60315663d657fd847c Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 14:58:15 -0800 Subject: [PATCH 03/42] Tweak --- .github/workflows/backend_selective_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend_selective_testing.yml b/.github/workflows/backend_selective_testing.yml index 3dc9e3d238fc55..cb6688775bcfc5 100644 --- a/.github/workflows/backend_selective_testing.yml +++ b/.github/workflows/backend_selective_testing.yml @@ -41,7 +41,7 @@ jobs: uses: actions/download-artifact@v4 with: path: .coverage - artifact-id: 4972973279 + artifact-ids: 4972973279 run-id: 20529759656 - name: Get changed files From 740ad75baf856cbd62f04e375edd318de9c2e89b Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 15:14:49 -0800 Subject: [PATCH 04/42] Tweak --- .github/workflows/backend_selective_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend_selective_testing.yml b/.github/workflows/backend_selective_testing.yml index cb6688775bcfc5..f263ed1915ae93 100644 --- a/.github/workflows/backend_selective_testing.yml +++ b/.github/workflows/backend_selective_testing.yml @@ -41,7 +41,7 @@ jobs: uses: actions/download-artifact@v4 with: path: .coverage - artifact-ids: 4972973279 + name: pycoverage-sqlite-combined-20529759656 run-id: 20529759656 - name: Get changed files From 2d4b6af411d2da5388b42a7e233a93a0d0a86128 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 15:27:13 -0800 Subject: [PATCH 05/42] Tweak --- .github/workflows/backend_selective_testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backend_selective_testing.yml b/.github/workflows/backend_selective_testing.yml index f263ed1915ae93..afeed6ccc86db8 100644 --- a/.github/workflows/backend_selective_testing.yml +++ b/.github/workflows/backend_selective_testing.yml @@ -38,10 +38,10 @@ jobs: # TODO: Gcloud - name: Download coverage database - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7 with: path: .coverage - name: pycoverage-sqlite-combined-20529759656 + name: 4972973279 run-id: 20529759656 - name: Get changed files From e0aca0365feb2cfe9d9bddf68793364782a42025 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 15:39:22 -0800 Subject: [PATCH 06/42] Tweak --- .github/workflows/backend_selective_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend_selective_testing.yml b/.github/workflows/backend_selective_testing.yml index afeed6ccc86db8..fdf18a0a8e99bf 100644 --- a/.github/workflows/backend_selective_testing.yml +++ b/.github/workflows/backend_selective_testing.yml @@ -41,7 +41,7 @@ jobs: uses: actions/download-artifact@v7 with: path: .coverage - name: 4972973279 + artifact-ids: 4972973279 run-id: 20529759656 - name: Get changed files From 38ba69be2723f412a5a77eda4c181e714f2039f5 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 15:50:41 -0800 Subject: [PATCH 07/42] Tweak --- .../workflows/backend_selective_testing.yml | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backend_selective_testing.yml b/.github/workflows/backend_selective_testing.yml index fdf18a0a8e99bf..413decb388a04c 100644 --- a/.github/workflows/backend_selective_testing.yml +++ b/.github/workflows/backend_selective_testing.yml @@ -36,13 +36,42 @@ jobs: with: mode: backend-ci - # TODO: Gcloud - - name: Download coverage database - uses: actions/download-artifact@v7 + # Download coverage artifact from a previous workflow run + # actions/download-artifact doesn't support cross-run downloads, so we use alternatives + + # Option 1: Use dawidd6/action-download-artifact (supports cross-run downloads) + - name: Download coverage database from run 20529759656 + uses: dawidd6/action-download-artifact@v6 + continue-on-error: true + id: download-coverage with: + run_id: 20529759656 + name: pycoverage-sqlite-combined-20529759656 path: .coverage - artifact-ids: 4972973279 - run-id: 20529759656 + github_token: ${{ github.token }} + + # Option 2: Fallback to manual download using gh CLI + - name: Manual download of coverage artifact + if: steps.download-coverage.outcome == 'failure' + env: + GH_TOKEN: ${{ github.token }} + run: | + echo "Downloading artifact 4972973279 (pycoverage-sqlite-combined-20529759656)..." + + # Download the artifact zip + gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/getsentry/sentry/actions/artifacts/4972973279/zip \ + > coverage.zip + + # Extract to .coverage directory + mkdir -p .coverage + unzip -q coverage.zip -d .coverage + rm coverage.zip + + echo "Coverage artifact downloaded successfully" + ls -la .coverage - name: Get changed files id: changed-files From 3f8b89637fea173f7c8671a5ab70367e2bc22eeb Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 16:53:42 -0800 Subject: [PATCH 08/42] Tweak --- .../workflows/backend_selective_testing.yml | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/.github/workflows/backend_selective_testing.yml b/.github/workflows/backend_selective_testing.yml index 413decb388a04c..c15bb2f1f0b237 100644 --- a/.github/workflows/backend_selective_testing.yml +++ b/.github/workflows/backend_selective_testing.yml @@ -50,29 +50,6 @@ jobs: path: .coverage github_token: ${{ github.token }} - # Option 2: Fallback to manual download using gh CLI - - name: Manual download of coverage artifact - if: steps.download-coverage.outcome == 'failure' - env: - GH_TOKEN: ${{ github.token }} - run: | - echo "Downloading artifact 4972973279 (pycoverage-sqlite-combined-20529759656)..." - - # Download the artifact zip - gh api \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - /repos/getsentry/sentry/actions/artifacts/4972973279/zip \ - > coverage.zip - - # Extract to .coverage directory - mkdir -p .coverage - unzip -q coverage.zip -d .coverage - rm coverage.zip - - echo "Coverage artifact downloaded successfully" - ls -la .coverage - - name: Get changed files id: changed-files uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 @@ -91,7 +68,7 @@ jobs: env: SELECTIVE_TESTING_ENABLED: true CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} - COVERAGE_DB_PATH: .coverage + COVERAGE_DB_PATH: .coverage/.coverage.combined - name: Inspect failure if: failure() From 9aabeb7eb0da7d8ee921f21bfebcd528f0382434 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 17:22:51 -0800 Subject: [PATCH 09/42] Tweak --- .github/workflows/backend_selective_testing.yml | 5 ++--- src/sentry/testutils/pytest/sentry.py | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backend_selective_testing.yml b/.github/workflows/backend_selective_testing.yml index c15bb2f1f0b237..9471e6c2cb7630 100644 --- a/.github/workflows/backend_selective_testing.yml +++ b/.github/workflows/backend_selective_testing.yml @@ -21,11 +21,10 @@ jobs: strategy: fail-fast: false matrix: - instance: - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21] + instance: [0] env: - MATRIX_INSTANCE_TOTAL: 22 + MATRIX_INSTANCE_TOTAL: 1 steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 diff --git a/src/sentry/testutils/pytest/sentry.py b/src/sentry/testutils/pytest/sentry.py index b594efad402bc0..e067fb9e10c770 100644 --- a/src/sentry/testutils/pytest/sentry.py +++ b/src/sentry/testutils/pytest/sentry.py @@ -419,6 +419,9 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item selected_items, discarded_items, affected_test_files = filter_items_by_coverage( items, changed_files, coverage_db_path ) + config.get_terminal_writer().line(f"Selected items: {selected_items}") + config.get_terminal_writer().line(f"Discarded items: {discarded_items}") + config.get_terminal_writer().line(f"Affected test files: {affected_test_files}") if affected_test_files is not None: config.get_terminal_writer().line( From 4258b128875b59c36f1eb77734af3990bda40749 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 17:45:14 -0800 Subject: [PATCH 10/42] Tweak --- src/sentry/testutils/pytest/sentry.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sentry/testutils/pytest/sentry.py b/src/sentry/testutils/pytest/sentry.py index e067fb9e10c770..7e202c32131d2d 100644 --- a/src/sentry/testutils/pytest/sentry.py +++ b/src/sentry/testutils/pytest/sentry.py @@ -416,6 +416,10 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item ) # Filter tests using coverage data + config.get_terminal_writer().line( + f"Filtering tests using coverage data from {coverage_db_path}" + ) + config.get_terminal_writer().line(f"Changed files: {changed_files}") selected_items, discarded_items, affected_test_files = filter_items_by_coverage( items, changed_files, coverage_db_path ) From e1b720efa7ff5933884ac1ca13f38270e8c4eed4 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 18:45:28 -0800 Subject: [PATCH 11/42] Tweak --- src/sentry/testutils/pytest/sentry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentry/testutils/pytest/sentry.py b/src/sentry/testutils/pytest/sentry.py index 7e202c32131d2d..603948028f4290 100644 --- a/src/sentry/testutils/pytest/sentry.py +++ b/src/sentry/testutils/pytest/sentry.py @@ -408,8 +408,8 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item coverage_db_path = os.environ.get("COVERAGE_DB_PATH", ".coverage.combined") if changed_files_str: - # Parse changed files from comma-separated string - changed_files = [f.strip() for f in changed_files_str.split(",") if f.strip()] + # Parse changed files from space-separated string + changed_files = [f.strip() for f in changed_files_str.split(" ") if f.strip()] config.get_terminal_writer().line( f"Selective testing enabled for {len(changed_files)} changed file(s)" From c0de6e6926d24d9cc547a2ad0f9dd454176afe7d Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 19:04:34 -0800 Subject: [PATCH 12/42] Tweak --- src/sentry/testutils/pytest/selective_testing.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index 472aaa10ddf7eb..1ce7ca5d553b16 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -1,3 +1,5 @@ +# flake8: noqa: S002 + from __future__ import annotations import os @@ -34,6 +36,9 @@ def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> s test_contexts = set() for file_path in source_files: + cleaned_file_path = file_path + if cleaned_file_path.startswith("/src"): + cleaned_file_path = cleaned_file_path[len("/src") :] # Query for test contexts that executed this file cur.execute( """ @@ -44,7 +49,7 @@ def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> s WHERE f.path LIKE '%' || ? AND c.context != '' """, - (f"/{file_path}",), + (f"{cleaned_file_path}",), ) for context, bitblob in cur.fetchall(): @@ -64,6 +69,7 @@ def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> s except (sqlite3.Error, Exception) as e: # Log the error but don't fail the test run + print(f"Warning: Could not query coverage database: {e}", file=sys.stderr) return None From 6d6d3c85e0e5d827716c96eb54bc1aa4d73d5254 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 19:22:18 -0800 Subject: [PATCH 13/42] Tweak --- src/sentry/testutils/pytest/selective_testing.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index 1ce7ca5d553b16..7ca328160dfa62 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -40,6 +40,8 @@ def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> s if cleaned_file_path.startswith("/src"): cleaned_file_path = cleaned_file_path[len("/src") :] # Query for test contexts that executed this file + + print(f"Querying coverage database for {cleaned_file_path}") cur.execute( """ SELECT c.context, lb.numbits @@ -52,8 +54,11 @@ def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> s (f"{cleaned_file_path}",), ) + print(f"Found {len(cur.fetchall())} contexts for {cleaned_file_path}") + for context, bitblob in cur.fetchall(): if _file_executed(bitblob): + print(f"Found executed context: {context}") test_contexts.add(context) conn.close() @@ -89,6 +94,7 @@ def filter_items_by_coverage(items, changed_files: list[str], coverage_db_path: or None if coverage data could not be loaded. """ affected_test_files = get_affected_tests_from_coverage(coverage_db_path, changed_files) + print(f"Affected test files: {affected_test_files}") if affected_test_files is None: # Could not load coverage data, return all items as selected From 9b9ddd235de9f9b71c2625c19ac6e7edebd45154 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 19:29:34 -0800 Subject: [PATCH 14/42] Tweak --- src/sentry/testutils/pytest/selective_testing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index 7ca328160dfa62..d89f445567024e 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -36,6 +36,8 @@ def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> s test_contexts = set() for file_path in source_files: + if file_path.endswith("sentry/testutils/pytest/sentry.py"): + continue cleaned_file_path = file_path if cleaned_file_path.startswith("/src"): cleaned_file_path = cleaned_file_path[len("/src") :] @@ -51,7 +53,7 @@ def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> s WHERE f.path LIKE '%' || ? AND c.context != '' """, - (f"{cleaned_file_path}",), + (f"%{cleaned_file_path}",), ) print(f"Found {len(cur.fetchall())} contexts for {cleaned_file_path}") From b5265d2aeda8232cdcad1ba5f7c754c62eaa5b55 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 19:39:46 -0800 Subject: [PATCH 15/42] Tweak --- src/sentry/testutils/pytest/selective_testing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index d89f445567024e..8a7b3376dfdb25 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -59,6 +59,8 @@ def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> s print(f"Found {len(cur.fetchall())} contexts for {cleaned_file_path}") for context, bitblob in cur.fetchall(): + print(f"Context: {context}") + print(f"Bitblob: {bitblob}") if _file_executed(bitblob): print(f"Found executed context: {context}") test_contexts.add(context) From 9ad854daf4e6e8d9c370e6e8a5205ce3baa6e0d8 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 19:54:45 -0800 Subject: [PATCH 16/42] Tweak --- src/sentry/testutils/pytest/selective_testing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index 8a7b3376dfdb25..7a162515e63ef7 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -56,9 +56,9 @@ def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> s (f"%{cleaned_file_path}",), ) - print(f"Found {len(cur.fetchall())} contexts for {cleaned_file_path}") - - for context, bitblob in cur.fetchall(): + contexts = cur.fetchall() + print(f"Found {len(contexts)} contexts for {cleaned_file_path}") + for context, bitblob in contexts: print(f"Context: {context}") print(f"Bitblob: {bitblob}") if _file_executed(bitblob): From bf7844453c6b65bbf0a5dae96ce31275a012b55d Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 20:03:37 -0800 Subject: [PATCH 17/42] Tweak --- src/sentry/testutils/pytest/selective_testing.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index 7a162515e63ef7..4a98ebd1abd5d5 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -57,11 +57,10 @@ def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> s ) contexts = cur.fetchall() + print(contexts) print(f"Found {len(contexts)} contexts for {cleaned_file_path}") for context, bitblob in contexts: - print(f"Context: {context}") - print(f"Bitblob: {bitblob}") - if _file_executed(bitblob): + if _file_executed(bytes(bitblob)): print(f"Found executed context: {context}") test_contexts.add(context) From 862cff19561b6cc9d2952f8dff2625e012f68b07 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 20:52:55 -0800 Subject: [PATCH 18/42] Tweak --- .../testutils/pytest/selective_testing.py | 88 ++++++------------- src/sentry/testutils/pytest/sentry.py | 15 +--- 2 files changed, 30 insertions(+), 73 deletions(-) diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index 4a98ebd1abd5d5..cd7538ce180eac 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -1,49 +1,40 @@ -# flake8: noqa: S002 - from __future__ import annotations import os import sqlite3 -import sys - - -def _file_executed(bitblob: bytes) -> bool: - """ - Returns True if any line in the file was executed (bitblob has any bits set). - """ - return any(b != 0 for b in bitblob) +from sentry.testutils import pytest -def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> set[str] | None: - """ - Query the coverage database to find which tests executed code in the given source files. +PYTEST_IGNORED_FILES = [ + # the pytest code itself is not part of the test suite but will be referenced by most tests + "sentry/testutils/pytest/sentry.py", +] - Args: - db_path: Path to the .coverage SQLite database - source_files: List of source file paths that have changed - Returns: - Set of test file paths (e.g., 'tests/sentry/api/test_foo.py'), - or None if the database doesn't exist or there's an error. - """ - if not os.path.exists(db_path): - return None +def filter_items_by_coverage( + config: pytest.Config, + items: list[pytest.Item], + changed_files: list[str], + coverage_db_path: str, +) -> tuple[list[pytest.Item], list[pytest.Item], set[str]]: + if not os.path.exists(coverage_db_path): + raise ValueError(f"Coverage database not found at {coverage_db_path}") + affected_test_files = set() try: - conn = sqlite3.connect(db_path) + conn = sqlite3.connect(coverage_db_path) cur = conn.cursor() test_contexts = set() - for file_path in source_files: - if file_path.endswith("sentry/testutils/pytest/sentry.py"): + for file_path in changed_files: + if any(file_path.endswith(ignored_file) for ignored_file in PYTEST_IGNORED_FILES): continue + cleaned_file_path = file_path if cleaned_file_path.startswith("/src"): cleaned_file_path = cleaned_file_path[len("/src") :] - # Query for test contexts that executed this file - print(f"Querying coverage database for {cleaned_file_path}") cur.execute( """ SELECT c.context, lb.numbits @@ -56,52 +47,25 @@ def get_affected_tests_from_coverage(db_path: str, source_files: list[str]) -> s (f"%{cleaned_file_path}",), ) - contexts = cur.fetchall() - print(contexts) - print(f"Found {len(contexts)} contexts for {cleaned_file_path}") - for context, bitblob in contexts: - if _file_executed(bytes(bitblob)): - print(f"Found executed context: {context}") + for context, bitblob in cur.fetchall(): + # Check if test was executed + if any(b != 0 for b in bytes(bitblob)): test_contexts.add(context) conn.close() # Extract test file paths from contexts - # Context format: 'tests/foo/bar.py::TestClass::test_function' + # Context format: 'tests/foo/bar.py::TestClass::test_function|run' test_files = set() for context in test_contexts: test_file = context.split("::", 1)[0] test_files.add(test_file) - return test_files - except (sqlite3.Error, Exception) as e: - # Log the error but don't fail the test run - - print(f"Warning: Could not query coverage database: {e}", file=sys.stderr) - return None - - -def filter_items_by_coverage(items, changed_files: list[str], coverage_db_path: str): - """ - Filter pytest items to only include tests affected by the changed files. - - Args: - items: List of pytest.Item objects to filter - changed_files: List of source files that have changed - coverage_db_path: Path to the coverage database - - Returns: - Tuple of (selected_items, discarded_items, affected_test_files) - where affected_test_files is the set of test files found in coverage data, - or None if coverage data could not be loaded. - """ - affected_test_files = get_affected_tests_from_coverage(coverage_db_path, changed_files) - print(f"Affected test files: {affected_test_files}") + raise ValueError(f"Could not query coverage database: {e}") - if affected_test_files is None: - # Could not load coverage data, return all items as selected - return list(items), [], None + config.get_terminal_writer().line(f"Found {len(affected_test_files)} affected test files") + config.get_terminal_writer().line(f"Affected test files: {affected_test_files}") # Filter items to only include tests from affected files selected_items = [] @@ -115,4 +79,4 @@ def filter_items_by_coverage(items, changed_files: list[str], coverage_db_path: else: discarded_items.append(item) - return selected_items, discarded_items, affected_test_files + return selected_items, discarded_items diff --git a/src/sentry/testutils/pytest/sentry.py b/src/sentry/testutils/pytest/sentry.py index 603948028f4290..b92d20784e4ade 100644 --- a/src/sentry/testutils/pytest/sentry.py +++ b/src/sentry/testutils/pytest/sentry.py @@ -404,10 +404,11 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item # Selective testing based on coverage data if os.environ.get("SELECTIVE_TESTING_ENABLED"): - changed_files_str = os.environ.get("CHANGED_FILES", "") + changed_files_str = os.environ.get("CHANGED_FILES", None) + # TODO coverage_db_path = os.environ.get("COVERAGE_DB_PATH", ".coverage.combined") - if changed_files_str: + if changed_files_str is not None: # Parse changed files from space-separated string changed_files = [f.strip() for f in changed_files_str.split(" ") if f.strip()] @@ -419,13 +420,9 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item config.get_terminal_writer().line( f"Filtering tests using coverage data from {coverage_db_path}" ) - config.get_terminal_writer().line(f"Changed files: {changed_files}") selected_items, discarded_items, affected_test_files = filter_items_by_coverage( - items, changed_files, coverage_db_path + config, items, changed_files, coverage_db_path ) - config.get_terminal_writer().line(f"Selected items: {selected_items}") - config.get_terminal_writer().line(f"Discarded items: {discarded_items}") - config.get_terminal_writer().line(f"Affected test files: {affected_test_files}") if affected_test_files is not None: config.get_terminal_writer().line( @@ -438,10 +435,6 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item # Update items with filtered list items[:] = selected_items initial_discard = discarded_items - else: - config.get_terminal_writer().line( - "Warning: Could not load coverage data, running all tests" - ) # Existing grouping logic (unchanged) total_groups = int(os.environ.get("TOTAL_TEST_GROUPS", 1)) From 78f129eb3a1b3f0805c8c9af2674affebbee1e49 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 21:00:01 -0800 Subject: [PATCH 19/42] Tweak --- src/sentry/testutils/pytest/selective_testing.py | 2 +- src/sentry/testutils/pytest/sentry.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index cd7538ce180eac..c6efcec191520d 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -79,4 +79,4 @@ def filter_items_by_coverage( else: discarded_items.append(item) - return selected_items, discarded_items + return selected_items, discarded_items, affected_test_files diff --git a/src/sentry/testutils/pytest/sentry.py b/src/sentry/testutils/pytest/sentry.py index b92d20784e4ade..51763e93fea4d5 100644 --- a/src/sentry/testutils/pytest/sentry.py +++ b/src/sentry/testutils/pytest/sentry.py @@ -405,7 +405,7 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item # Selective testing based on coverage data if os.environ.get("SELECTIVE_TESTING_ENABLED"): changed_files_str = os.environ.get("CHANGED_FILES", None) - # TODO + # TODO: Remove default value coverage_db_path = os.environ.get("COVERAGE_DB_PATH", ".coverage.combined") if changed_files_str is not None: From fc23c8fc2345eb1f8b76eff97dd9699bf4c9a316 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Mon, 29 Dec 2025 21:09:06 -0800 Subject: [PATCH 20/42] Tweak --- src/sentry/testutils/pytest/selective_testing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index c6efcec191520d..7b3dc156f2c117 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -56,10 +56,9 @@ def filter_items_by_coverage( # Extract test file paths from contexts # Context format: 'tests/foo/bar.py::TestClass::test_function|run' - test_files = set() for context in test_contexts: test_file = context.split("::", 1)[0] - test_files.add(test_file) + affected_test_files.add(test_file) except (sqlite3.Error, Exception) as e: raise ValueError(f"Could not query coverage database: {e}") From 0b26c0835f1c182c34ae071207b75d5d2865f5a9 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Tue, 30 Dec 2025 11:20:26 -0800 Subject: [PATCH 21/42] Updates --- .../workflows/backend_selective_testing.yml | 39 +++++++++++++------ .../testutils/pytest/selective_testing.py | 4 -- src/sentry/testutils/pytest/sentry.py | 9 ++++- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/.github/workflows/backend_selective_testing.yml b/.github/workflows/backend_selective_testing.yml index 9471e6c2cb7630..0e139677b7c324 100644 --- a/.github/workflows/backend_selective_testing.yml +++ b/.github/workflows/backend_selective_testing.yml @@ -1,4 +1,4 @@ -name: backend - selective +name: backend (selective) on: pull_request: @@ -10,8 +10,32 @@ concurrency: cancel-in-progress: true jobs: + files-changed: + name: detect what files changed + runs-on: ubuntu-24.04 + timeout-minutes: 3 + # Map a step output to a job output + outputs: + api_docs: ${{ steps.changes.outputs.api_docs }} + backend: ${{ steps.changes.outputs.backend_all }} + backend_dependencies: ${{ steps.changes.outputs.backend_dependencies }} + backend_api_urls: ${{ steps.changes.outputs.backend_api_urls }} + backend_any_type: ${{ steps.changes.outputs.backend_any_type }} + migration_lockfile: ${{ steps.changes.outputs.migration_lockfile }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for backend file changes + uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + backend-test-selective: - name: backend test (selective) + if: needs.files-changed.outputs.backend == 'true' + needs: files-changed + name: backend test runs-on: ubuntu-24.04 timeout-minutes: 60 permissions: @@ -35,10 +59,9 @@ jobs: with: mode: backend-ci + # TODO: Replace with Gcloud download # Download coverage artifact from a previous workflow run # actions/download-artifact doesn't support cross-run downloads, so we use alternatives - - # Option 1: Use dawidd6/action-download-artifact (supports cross-run downloads) - name: Download coverage database from run 20529759656 uses: dawidd6/action-download-artifact@v6 continue-on-error: true @@ -53,14 +76,6 @@ jobs: id: changed-files uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 - - name: List all changed files - env: - ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} - run: | - for file in ${ALL_CHANGED_FILES}; do - echo "$file was changed" - done - - name: Run backend tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) id: run_backend_tests run: make test-python-ci diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index 7b3dc156f2c117..392fee6b47b6a9 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import sqlite3 from sentry.testutils import pytest @@ -17,9 +16,6 @@ def filter_items_by_coverage( changed_files: list[str], coverage_db_path: str, ) -> tuple[list[pytest.Item], list[pytest.Item], set[str]]: - if not os.path.exists(coverage_db_path): - raise ValueError(f"Coverage database not found at {coverage_db_path}") - affected_test_files = set() try: conn = sqlite3.connect(coverage_db_path) diff --git a/src/sentry/testutils/pytest/sentry.py b/src/sentry/testutils/pytest/sentry.py index 51763e93fea4d5..4d6db03f127b96 100644 --- a/src/sentry/testutils/pytest/sentry.py +++ b/src/sentry/testutils/pytest/sentry.py @@ -405,8 +405,13 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item # Selective testing based on coverage data if os.environ.get("SELECTIVE_TESTING_ENABLED"): changed_files_str = os.environ.get("CHANGED_FILES", None) - # TODO: Remove default value - coverage_db_path = os.environ.get("COVERAGE_DB_PATH", ".coverage.combined") + + coverage_db_path = os.environ.get("COVERAGE_DB_PATH", None) + if coverage_db_path is None: + raise ValueError("COVERAGE_DB_PATH is not set") + + if not os.path.exists(coverage_db_path): + raise ValueError(f"Coverage database not found at {coverage_db_path}") if changed_files_str is not None: # Parse changed files from space-separated string From 25c469301427cb154c74ebc3d3c5a6567eb41fe9 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Wed, 14 Jan 2026 21:45:39 -0800 Subject: [PATCH 22/42] Updates --- .github/workflows/backend-selective.yml | 164 ++++++++++++++++++ .../workflows/backend_selective_testing.yml | 103 ----------- .../testutils/pytest/selective_testing.py | 2 +- src/sentry/testutils/pytest/sentry.py | 61 +++---- 4 files changed, 189 insertions(+), 141 deletions(-) create mode 100644 .github/workflows/backend-selective.yml delete mode 100644 .github/workflows/backend_selective_testing.yml diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml new file mode 100644 index 00000000000000..8b0c8b4c49e69a --- /dev/null +++ b/.github/workflows/backend-selective.yml @@ -0,0 +1,164 @@ +name: backend (selective) + +on: + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + files-changed: + name: detect what files changed + runs-on: ubuntu-24.04 + timeout-minutes: 3 + # Map a step output to a job output + outputs: + api_docs: ${{ steps.changes.outputs.api_docs }} + backend: ${{ steps.changes.outputs.backend_all }} + backend_dependencies: ${{ steps.changes.outputs.backend_dependencies }} + backend_api_urls: ${{ steps.changes.outputs.backend_api_urls }} + backend_any_type: ${{ steps.changes.outputs.backend_any_type }} + migration_lockfile: ${{ steps.changes.outputs.migration_lockfile }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for backend file changes + uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + backend-test-selective: + if: needs.files-changed.outputs.backend == 'true' + needs: files-changed + name: backend test + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: read + id-token: write + actions: read # used for DIM metadata + strategy: + fail-fast: false + matrix: + instance: [0] + + env: + MATRIX_INSTANCE_TOTAL: 1 + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 # Need full history for git diff + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + mode: backend-ci + + - name: Authenticate to Google Cloud + id: gcloud-auth + uses: google-github-actions/auth@v2 + with: + project_id: sentry-dev-tooling + workload_identity_provider: ${{ secrets.SENTRY_GCP_DEV_WORKLOAD_IDENTITY_POOL }} + service_account: ${{ secrets.COLLECT_TEST_DATA_SERVICE_ACCOUNT_EMAIL }} + + - name: Find coverage data for selective testing + id: find-coverage + env: + GCS_BUCKET: sentry-coverage-data + run: | + set -euo pipefail + + # Get the base commit (what the PR branches from) + BASE_SHA="${{ github.event.pull_request.base.sha }}" + + echo "Looking for coverage data starting from base commit: $BASE_SHA" + + COVERAGE_SHA="" + for sha in $(git rev-list "$BASE_SHA" --max-count=30); do + # Check if coverage exists in GCS for this commit + if gcloud storage ls "gs://${GCS_BUCKET}/${sha}/" &>/dev/null; then + COVERAGE_SHA="$sha" + echo "Found coverage data at commit: $sha" + break + fi + echo "No coverage at $sha, checking parent..." + done + + if [[ -z "$COVERAGE_SHA" ]]; then + echo "No coverage found in last 30 commits, will run full test suite" + echo "found=false" >> "$GITHUB_OUTPUT" + else + echo "found=true" >> "$GITHUB_OUTPUT" + echo "coverage-sha=$COVERAGE_SHA" >> "$GITHUB_OUTPUT" + fi + + - name: Download coverage database + id: download-coverage + if: steps.find-coverage.outputs.found == 'true' + env: + COVERAGE_SHA: ${{ steps.find-coverage.outputs.coverage-sha }} + run: | + set -euxo pipefail + mkdir -p .coverage + gcloud storage cp "gs://sentry-coverage-data/${COVERAGE_SHA}/*" .coverage/ || true + + # Find the coverage file (could be .coverage.* format) + COVERAGE_FILE=$(ls .coverage/.coverage.* 2>/dev/null | head -1 || true) + if [[ -z "$COVERAGE_FILE" ]]; then + echo "Warning: No coverage file found in downloaded data, will run full test suite" + ls -la .coverage/ || true + echo "coverage-file=" >> "$GITHUB_OUTPUT" + else + echo "Downloaded coverage file: $COVERAGE_FILE" + echo "coverage-file=$COVERAGE_FILE" >> "$GITHUB_OUTPUT" + fi + + - name: Get changed files + id: changed-files + run: | + # Get files changed between base and head of PR + BASE_SHA="${{ github.event.pull_request.base.sha }}" + HEAD_SHA="${{ github.event.pull_request.head.sha }}" + + CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" | tr '\n' ' ') + echo "Changed files: $CHANGED_FILES" + echo "files=$CHANGED_FILES" >> "$GITHUB_OUTPUT" + + - name: Run backend tests (selective) + if: steps.download-coverage.outputs.coverage-file != '' + id: run_backend_tests_selective + run: make test-python-ci + env: + CHANGED_FILES: ${{ steps.changed-files.outputs.files }} + COVERAGE_DB_PATH: ${{ steps.download-coverage.outputs.coverage-file }} + + - name: Run backend tests (full - no coverage data) + if: steps.download-coverage.outputs.coverage-file == '' + id: run_backend_tests_full + run: make test-python-ci + + - name: Inspect failure + if: failure() + run: | + if command -v devservices; then + devservices logs + fi + + - name: Collect test data + uses: ./.github/actions/collect-test-data + if: ${{ !cancelled() }} + with: + artifact_path: .artifacts/pytest.json # TODO + gcs_bucket: ${{ secrets.COLLECT_TEST_DATA_GCS_BUCKET }} + gcp_project_id: ${{ secrets.COLLECT_TEST_DATA_GCP_PROJECT_ID }} + workload_identity_provider: ${{ secrets.SENTRY_GCP_DEV_WORKLOAD_IDENTITY_POOL }} + service_account_email: ${{ secrets.COLLECT_TEST_DATA_SERVICE_ACCOUNT_EMAIL }} + matrix_instance_number: ${{ steps.setup.outputs.matrix-instance-number }} diff --git a/.github/workflows/backend_selective_testing.yml b/.github/workflows/backend_selective_testing.yml deleted file mode 100644 index 0e139677b7c324..00000000000000 --- a/.github/workflows/backend_selective_testing.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: backend (selective) - -on: - pull_request: - -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - files-changed: - name: detect what files changed - runs-on: ubuntu-24.04 - timeout-minutes: 3 - # Map a step output to a job output - outputs: - api_docs: ${{ steps.changes.outputs.api_docs }} - backend: ${{ steps.changes.outputs.backend_all }} - backend_dependencies: ${{ steps.changes.outputs.backend_dependencies }} - backend_api_urls: ${{ steps.changes.outputs.backend_api_urls }} - backend_any_type: ${{ steps.changes.outputs.backend_any_type }} - migration_lockfile: ${{ steps.changes.outputs.migration_lockfile }} - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - name: Check for backend file changes - uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 - id: changes - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - - backend-test-selective: - if: needs.files-changed.outputs.backend == 'true' - needs: files-changed - name: backend test - runs-on: ubuntu-24.04 - timeout-minutes: 60 - permissions: - contents: read - id-token: write - actions: read # used for DIM metadata - strategy: - fail-fast: false - matrix: - instance: [0] - - env: - MATRIX_INSTANCE_TOTAL: 1 - - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - mode: backend-ci - - # TODO: Replace with Gcloud download - # Download coverage artifact from a previous workflow run - # actions/download-artifact doesn't support cross-run downloads, so we use alternatives - - name: Download coverage database from run 20529759656 - uses: dawidd6/action-download-artifact@v6 - continue-on-error: true - id: download-coverage - with: - run_id: 20529759656 - name: pycoverage-sqlite-combined-20529759656 - path: .coverage - github_token: ${{ github.token }} - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 - - - name: Run backend tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) - id: run_backend_tests - run: make test-python-ci - env: - SELECTIVE_TESTING_ENABLED: true - CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} - COVERAGE_DB_PATH: .coverage/.coverage.combined - - - name: Inspect failure - if: failure() - run: | - if command -v devservices; then - devservices logs - fi - - # - name: Collect test data - # uses: ./.github/actions/collect-test-data - # if: ${{ !cancelled() }} - # with: - # artifact_path: .artifacts/pytest.json # TODO - # gcs_bucket: ${{ secrets.COLLECT_TEST_DATA_GCS_BUCKET }} - # gcp_project_id: ${{ secrets.COLLECT_TEST_DATA_GCP_PROJECT_ID }} - # workload_identity_provider: ${{ secrets.SENTRY_GCP_DEV_WORKLOAD_IDENTITY_POOL }} - # service_account_email: ${{ secrets.COLLECT_TEST_DATA_SERVICE_ACCOUNT_EMAIL }} - # matrix_instance_number: ${{ steps.setup.outputs.matrix-instance-number }} diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index 392fee6b47b6a9..46c8c425515e0c 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -2,7 +2,7 @@ import sqlite3 -from sentry.testutils import pytest +import pytest PYTEST_IGNORED_FILES = [ # the pytest code itself is not part of the test suite but will be referenced by most tests diff --git a/src/sentry/testutils/pytest/sentry.py b/src/sentry/testutils/pytest/sentry.py index 4d6db03f127b96..b9aaa86eb00758 100644 --- a/src/sentry/testutils/pytest/sentry.py +++ b/src/sentry/testutils/pytest/sentry.py @@ -18,7 +18,6 @@ from sentry.runner.importer import install_plugin_apps from sentry.silo.base import SiloMode -from sentry.testutils.pytest.selective_testing import filter_items_by_coverage from sentry.testutils.region import TestEnvRegionDirectory from sentry.testutils.silo import monkey_patch_single_process_silo_mode_state from sentry.types import region @@ -400,48 +399,39 @@ def _shuffle_d(dct: dict[K, V]) -> dict[K, V]: def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: """After collection, we need to select tests based on group and group strategy""" - initial_discard = [] + # Selective test filtering based on coverage data + # If COVERAGE_DB_PATH and CHANGED_FILES are set, filter to tests that cover those files + coverage_db_path = os.environ.get("COVERAGE_DB_PATH") + changed_files_str = os.environ.get("CHANGED_FILES") - # Selective testing based on coverage data - if os.environ.get("SELECTIVE_TESTING_ENABLED"): - changed_files_str = os.environ.get("CHANGED_FILES", None) + if coverage_db_path and changed_files_str: + from sentry.testutils.pytest.selective_testing import filter_items_by_coverage - coverage_db_path = os.environ.get("COVERAGE_DB_PATH", None) - if coverage_db_path is None: - raise ValueError("COVERAGE_DB_PATH is not set") + changed_files = [f.strip() for f in changed_files_str.split() if f.strip()] - if not os.path.exists(coverage_db_path): - raise ValueError(f"Coverage database not found at {coverage_db_path}") - - if changed_files_str is not None: - # Parse changed files from space-separated string - changed_files = [f.strip() for f in changed_files_str.split(" ") if f.strip()] + if changed_files: + original_count = len(items) + try: + selected_items, deselected_items, _ = filter_items_by_coverage( + config=config, + items=items, + changed_files=changed_files, + coverage_db_path=coverage_db_path, + ) - config.get_terminal_writer().line( - f"Selective testing enabled for {len(changed_files)} changed file(s)" - ) + items[:] = selected_items - # Filter tests using coverage data - config.get_terminal_writer().line( - f"Filtering tests using coverage data from {coverage_db_path}" - ) - selected_items, discarded_items, affected_test_files = filter_items_by_coverage( - config, items, changed_files, coverage_db_path - ) + if deselected_items: + config.hook.pytest_deselected(items=deselected_items) - if affected_test_files is not None: config.get_terminal_writer().line( - f"Found {len(affected_test_files)} affected test file(s) from coverage data" + f"Selective testing: {len(items)}/{original_count} tests selected based on coverage" ) + except ValueError as e: config.get_terminal_writer().line( - f"Selected {len(selected_items)}/{len(items)} tests based on coverage" + f"Warning: Selective testing failed ({e}), running all tests" ) - # Update items with filtered list - items[:] = selected_items - initial_discard = discarded_items - - # Existing grouping logic (unchanged) total_groups = int(os.environ.get("TOTAL_TEST_GROUPS", 1)) current_group = int(os.environ.get("TEST_GROUP", 0)) grouping_strategy = os.environ.get("TEST_GROUP_STRATEGY", "scope") @@ -474,12 +464,9 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item config.get_terminal_writer().line(f"SENTRY_SHUFFLE_TESTS_SEED: {seed}") _shuffle(items, random.Random(seed)) - # Combine discards from both selective testing and grouping - all_discarded = initial_discard + discard - # This only needs to be done if there are items to be de-selected - if len(all_discarded) > 0: - config.hook.pytest_deselected(items=all_discarded) + if len(discard) > 0: + config.hook.pytest_deselected(items=discard) def pytest_xdist_setupnodes() -> None: From 7b566a7634655440aca83ea37c30effd1407134f Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Wed, 14 Jan 2026 22:04:40 -0800 Subject: [PATCH 23/42] Updates --- .github/workflows/backend-selective.yml | 153 ++++++++++++++---- .../scripts/calculate-backend-test-shards.py | 32 +++- .../scripts/compute-selected-tests.py | 119 ++++++++++++++ .../testutils/pytest/selective_testing.py | 7 +- 4 files changed, 274 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/scripts/compute-selected-tests.py diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml index 8b0c8b4c49e69a..aca074fb91ead1 100644 --- a/.github/workflows/backend-selective.yml +++ b/.github/workflows/backend-selective.yml @@ -9,12 +9,16 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 + SNUBA_NO_WORKERS: 1 + jobs: files-changed: name: detect what files changed runs-on: ubuntu-24.04 timeout-minutes: 3 - # Map a step output to a job output outputs: api_docs: ${{ steps.changes.outputs.api_docs }} backend: ${{ steps.changes.outputs.backend_all }} @@ -32,34 +36,30 @@ jobs: token: ${{ github.token }} filters: .github/file-filters.yml - backend-test-selective: + prepare-selective-tests: if: needs.files-changed.outputs.backend == 'true' needs: files-changed - name: backend test + name: prepare selective tests runs-on: ubuntu-24.04 - timeout-minutes: 60 + timeout-minutes: 10 permissions: contents: read id-token: write - actions: read # used for DIM metadata - strategy: - fail-fast: false - matrix: - instance: [0] - - env: - MATRIX_INSTANCE_TOTAL: 1 - + outputs: + has-coverage: ${{ steps.find-coverage.outputs.found }} + coverage-sha: ${{ steps.find-coverage.outputs.coverage-sha }} + changed-files: ${{ steps.changed-files.outputs.files }} + test-count: ${{ steps.compute-tests.outputs.test-count }} + has-selected-tests: ${{ steps.compute-tests.outputs.has-selected-tests }} steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: fetch-depth: 0 # Need full history for git diff - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup + - name: Setup Python + uses: actions/setup-python@v5 with: - mode: backend-ci + python-version: '3.13.1' - name: Authenticate to Google Cloud id: gcloud-auth @@ -113,7 +113,7 @@ jobs: # Find the coverage file (could be .coverage.* format) COVERAGE_FILE=$(ls .coverage/.coverage.* 2>/dev/null | head -1 || true) if [[ -z "$COVERAGE_FILE" ]]; then - echo "Warning: No coverage file found in downloaded data, will run full test suite" + echo "Warning: No coverage file found in downloaded data" ls -la .coverage/ || true echo "coverage-file=" >> "$GITHUB_OUTPUT" else @@ -132,18 +132,117 @@ jobs: echo "Changed files: $CHANGED_FILES" echo "files=$CHANGED_FILES" >> "$GITHUB_OUTPUT" - - name: Run backend tests (selective) + - name: Compute selected tests + id: compute-tests if: steps.download-coverage.outputs.coverage-file != '' - id: run_backend_tests_selective - run: make test-python-ci + run: | + python3 .github/workflows/scripts/compute-selected-tests.py \ + --coverage-db "${{ steps.download-coverage.outputs.coverage-file }}" \ + --changed-files "${{ steps.changed-files.outputs.files }}" \ + --output .artifacts/selected-tests.txt \ + --github-output + + - name: Upload coverage database artifact + if: steps.download-coverage.outputs.coverage-file != '' + uses: actions/upload-artifact@v4 + with: + name: coverage-db-${{ github.run_id }} + path: .coverage/ + retention-days: 1 + include-hidden-files: true + + - name: Upload selected tests artifact + if: steps.compute-tests.outputs.has-selected-tests == 'true' + uses: actions/upload-artifact@v4 + with: + name: selected-tests-${{ github.run_id }} + path: .artifacts/selected-tests.txt + retention-days: 1 + + calculate-shards: + if: needs.files-changed.outputs.backend == 'true' + needs: [files-changed, prepare-selective-tests] + name: calculate test shards + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + shard-count: ${{ steps.calculate-shards.outputs.shard-count }} + shard-indices: ${{ steps.calculate-shards.outputs.shard-indices }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + mode: backend-ci + skip-devservices: true + + - name: Download selected tests artifact + if: needs.prepare-selective-tests.outputs.has-selected-tests == 'true' + uses: actions/download-artifact@v4 + with: + name: selected-tests-${{ github.run_id }} + path: .artifacts/ + + - name: Calculate test shards + id: calculate-shards env: - CHANGED_FILES: ${{ steps.changed-files.outputs.files }} - COVERAGE_DB_PATH: ${{ steps.download-coverage.outputs.coverage-file }} + SELECTED_TESTS_FILE: ${{ needs.prepare-selective-tests.outputs.has-selected-tests == 'true' && '.artifacts/selected-tests.txt' || '' }} + SELECTED_TEST_COUNT: ${{ needs.prepare-selective-tests.outputs.test-count }} + run: | + python3 .github/workflows/scripts/calculate-backend-test-shards.py + + backend-test-selective: + if: needs.files-changed.outputs.backend == 'true' + needs: [files-changed, prepare-selective-tests, calculate-shards] + name: backend test + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: read + id-token: write + actions: read + strategy: + fail-fast: false + matrix: + instance: ${{ fromJSON(needs.calculate-shards.outputs.shard-indices) }} + + env: + MATRIX_INSTANCE_TOTAL: ${{ needs.calculate-shards.outputs.shard-count }} + TEST_GROUP_STRATEGY: roundrobin + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 - - name: Run backend tests (full - no coverage data) - if: steps.download-coverage.outputs.coverage-file == '' - id: run_backend_tests_full + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + mode: backend-ci + + - name: Download coverage database artifact + if: needs.prepare-selective-tests.outputs.has-coverage == 'true' + uses: actions/download-artifact@v4 + with: + name: coverage-db-${{ github.run_id }} + path: .coverage/ + + - name: Find coverage file + id: find-coverage-file + if: needs.prepare-selective-tests.outputs.has-coverage == 'true' + run: | + COVERAGE_FILE=$(ls .coverage/.coverage.* 2>/dev/null | head -1 || true) + echo "coverage-file=$COVERAGE_FILE" >> "$GITHUB_OUTPUT" + + - name: Run backend tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) + id: run_backend_tests run: make test-python-ci + env: + CHANGED_FILES: ${{ needs.prepare-selective-tests.outputs.changed-files }} + COVERAGE_DB_PATH: ${{ steps.find-coverage-file.outputs.coverage-file }} - name: Inspect failure if: failure() @@ -156,7 +255,7 @@ jobs: uses: ./.github/actions/collect-test-data if: ${{ !cancelled() }} with: - artifact_path: .artifacts/pytest.json # TODO + artifact_path: .artifacts/pytest.json gcs_bucket: ${{ secrets.COLLECT_TEST_DATA_GCS_BUCKET }} gcp_project_id: ${{ secrets.COLLECT_TEST_DATA_GCP_PROJECT_ID }} workload_identity_provider: ${{ secrets.SENTRY_GCP_DEV_WORKLOAD_IDENTITY_POOL }} diff --git a/.github/workflows/scripts/calculate-backend-test-shards.py b/.github/workflows/scripts/calculate-backend-test-shards.py index fca159736127d4..13049589716796 100755 --- a/.github/workflows/scripts/calculate-backend-test-shards.py +++ b/.github/workflows/scripts/calculate-backend-test-shards.py @@ -5,17 +5,20 @@ import re import subprocess import sys +from pathlib import Path TESTS_PER_SHARD = 1200 MIN_SHARDS = 1 MAX_SHARDS = 22 DEFAULT_SHARDS = 22 -PYTEST_ARGS = [ +PYTEST_BASE_ARGS = [ "pytest", "--collect-only", "--quiet", - "tests", +] + +PYTEST_IGNORE_ARGS = [ "--ignore=tests/acceptance", "--ignore=tests/apidocs", "--ignore=tests/js", @@ -24,9 +27,30 @@ def collect_test_count(): + """Collect test count, optionally filtering to selected test files.""" + selected_tests_file = os.environ.get("SELECTED_TESTS_FILE") + + if selected_tests_file: + path = Path(selected_tests_file) + if not path.exists(): + print(f"Selected tests file not found: {selected_tests_file}", file=sys.stderr) + return None + + with path.open() as f: + selected_files = [line.strip() for line in f if line.strip()] + + if not selected_files: + print("No selected test files, running 0 tests", file=sys.stderr) + return 0 + + print(f"Counting tests in {len(selected_files)} selected files", file=sys.stderr) + pytest_args = PYTEST_BASE_ARGS + selected_files + else: + pytest_args = PYTEST_BASE_ARGS + ["tests"] + PYTEST_IGNORE_ARGS + try: result = subprocess.run( - PYTEST_ARGS, + pytest_args, capture_output=True, text=True, check=False, @@ -40,7 +64,6 @@ def collect_test_count(): print(f"Collected {count} tests", file=sys.stderr) return count - # If no match, check if pytest failed if result.returncode != 0: print( f"Pytest collection failed (exit {result.returncode})", @@ -85,7 +108,6 @@ def calculate_shards(test_count): def main(): test_count = collect_test_count() shard_count = calculate_shards(test_count) - # Generate a JSON array of shard indices [0, 1, 2, ..., shard_count-1] shard_indices = json.dumps(list(range(shard_count))) github_output = os.getenv("GITHUB_OUTPUT") diff --git a/.github/workflows/scripts/compute-selected-tests.py b/.github/workflows/scripts/compute-selected-tests.py new file mode 100644 index 00000000000000..a4e104ad65ba31 --- /dev/null +++ b/.github/workflows/scripts/compute-selected-tests.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +""" +Compute selected tests based on coverage data and changed files. + +This script queries a coverage database to find which test files cover +the changed source files, outputting the list for selective test runs. +""" +from __future__ import annotations + +import argparse +import os +import sqlite3 +import sys +from pathlib import Path + +PYTEST_IGNORED_FILES = [ + "sentry/testutils/pytest/sentry.py", +] + + +def get_affected_test_files(coverage_db_path: str, changed_files: list[str]) -> set[str]: + """Query coverage DB to find test files that cover the changed source files.""" + affected_test_files: set[str] = set() + + conn = sqlite3.connect(coverage_db_path) + cur = conn.cursor() + + test_contexts: set[str] = set() + + for file_path in changed_files: + if any(file_path.endswith(ignored_file) for ignored_file in PYTEST_IGNORED_FILES): + continue + + cleaned_file_path = file_path + if cleaned_file_path.startswith("/src"): + cleaned_file_path = cleaned_file_path[len("/src") :] + + cur.execute( + """ + SELECT c.context, lb.numbits + FROM line_bits lb + JOIN file f ON lb.file_id = f.id + JOIN context c ON lb.context_id = c.id + WHERE f.path LIKE '%' || ? + AND c.context != '' + """, + (f"%{cleaned_file_path}",), + ) + + for context, bitblob in cur.fetchall(): + if any(b != 0 for b in bytes(bitblob)): + test_contexts.add(context) + + conn.close() + + # Extract test file paths from contexts + # Context format: 'tests/foo/bar.py::TestClass::test_function|run' + for context in test_contexts: + test_file = context.split("::", 1)[0] + affected_test_files.add(test_file) + + return affected_test_files + + +def main() -> int: + parser = argparse.ArgumentParser(description="Compute selected tests from coverage data") + parser.add_argument("--coverage-db", required=True, help="Path to coverage SQLite database") + parser.add_argument( + "--changed-files", required=True, help="Space-separated list of changed files" + ) + parser.add_argument("--output", help="Output file path for selected test files (one per line)") + parser.add_argument("--github-output", action="store_true", help="Write to GITHUB_OUTPUT") + args = parser.parse_args() + + coverage_db = Path(args.coverage_db) + if not coverage_db.exists(): + print(f"Error: Coverage database not found: {coverage_db}", file=sys.stderr) + return 1 + + changed_files = [f.strip() for f in args.changed_files.split() if f.strip()] + if not changed_files: + print("No changed files provided, selecting all tests") + affected_test_files: set[str] = set() + else: + print(f"Computing selected tests for {len(changed_files)} changed files...") + try: + affected_test_files = get_affected_test_files(str(coverage_db), changed_files) + except sqlite3.Error as e: + print(f"Error querying coverage database: {e}", file=sys.stderr) + return 1 + + print(f"Found {len(affected_test_files)} affected test files") + + if args.output: + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w") as f: + for test_file in sorted(affected_test_files): + f.write(f"{test_file}\n") + print(f"Wrote selected tests to {output_path}") + + if args.github_output: + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with open(github_output, "a") as f: + f.write(f"test-count={len(affected_test_files)}\n") + f.write(f"has-selected-tests={'true' if affected_test_files else 'false'}\n") + print(f"Wrote to GITHUB_OUTPUT: test-count={len(affected_test_files)}") + + if affected_test_files: + print("\nAffected test files:") + for test_file in sorted(affected_test_files): + print(f" {test_file}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py index 46c8c425515e0c..f284d6232c7d19 100644 --- a/src/sentry/testutils/pytest/selective_testing.py +++ b/src/sentry/testutils/pytest/selective_testing.py @@ -44,7 +44,6 @@ def filter_items_by_coverage( ) for context, bitblob in cur.fetchall(): - # Check if test was executed if any(b != 0 for b in bytes(bitblob)): test_contexts.add(context) @@ -56,18 +55,16 @@ def filter_items_by_coverage( test_file = context.split("::", 1)[0] affected_test_files.add(test_file) - except (sqlite3.Error, Exception) as e: - raise ValueError(f"Could not query coverage database: {e}") + except Exception as e: + raise ValueError(f"Could not query coverage database: {e}") from e config.get_terminal_writer().line(f"Found {len(affected_test_files)} affected test files") config.get_terminal_writer().line(f"Affected test files: {affected_test_files}") - # Filter items to only include tests from affected files selected_items = [] discarded_items = [] for item in items: - # Extract test file path from nodeid (e.g., 'tests/foo.py::TestClass::test_func') test_file = item.nodeid.split("::", 1)[0] if test_file in affected_test_files: selected_items.append(item) From 917abe33b8d451ee2d504b0d66ff745e3cb61651 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Thu, 15 Jan 2026 09:14:13 -0800 Subject: [PATCH 24/42] debug --- .github/workflows/development-environment.yml | 71 ------------------- .../scripts/compute-selected-tests.py | 10 +++ 2 files changed, 10 insertions(+), 71 deletions(-) delete mode 100644 .github/workflows/development-environment.yml diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml deleted file mode 100644 index 42804f51630dda..00000000000000 --- a/.github/workflows/development-environment.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: dev env -on: - pull_request: - paths: - - '.pre-commit-config.yaml' - - 'Makefile' - - '.github/workflows/development-environment.yml' - - 'requirements-*.txt' - - 'pyproject.toml' - - 'uv.lock' - - '.python-version' - - '.envrc' - - 'Brewfile' - - 'scripts/**' - - 'tools/**' - - 'src/sentry/runner/commands/devserver.py' - - 'src/sentry/runner/commands/devservices.py' - - 'bin/load-mocks' - -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 -env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 - -jobs: - test: - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - uses: astral-sh/setup-uv@884ad927a57e558e7a70b92f2bccf9198a4be546 # v6 - with: - version: '0.8.2' - # we just cache the venv-dir directly in action-setup-venv - enable-cache: false - - - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 - with: - cache-dependency-path: uv.lock - install-cmd: uv sync --only-dev --frozen --active - - - name: test-tools - run: make test-tools - - devenv: - runs-on: ubuntu-24.04 - timeout-minutes: 10 - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - uses: astral-sh/setup-uv@884ad927a57e558e7a70b92f2bccf9198a4be546 # v6 - with: - version: '0.8.2' - # we just cache the venv-dir directly in action-setup-venv - enable-cache: false - - - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 - with: - cache-dependency-path: uv.lock - # technically we can just use --only-dev but more cache is nice - install-cmd: uv sync --frozen --active - - - name: devenv sync - run: | - devenv --nocoderoot sync diff --git a/.github/workflows/scripts/compute-selected-tests.py b/.github/workflows/scripts/compute-selected-tests.py index a4e104ad65ba31..1a8b13bbe03cef 100644 --- a/.github/workflows/scripts/compute-selected-tests.py +++ b/.github/workflows/scripts/compute-selected-tests.py @@ -25,6 +25,16 @@ def get_affected_test_files(coverage_db_path: str, changed_files: list[str]) -> conn = sqlite3.connect(coverage_db_path) cur = conn.cursor() + # Verify required tables exist (need context tracking enabled) + tables = { + r[0] for r in cur.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() + } + if "line_bits" not in tables or "context" not in tables: + raise ValueError( + "Coverage database missing line_bits/context tables. " + "Coverage must be collected with --cov-context=test" + ) + test_contexts: set[str] = set() for file_path in changed_files: From 6a39767eff588e0e2eaed4520c9eec517a6fbc4e Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Thu, 15 Jan 2026 09:20:56 -0800 Subject: [PATCH 25/42] Iterate --- .github/workflows/backend-selective.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml index aca074fb91ead1..f5107b3cd01c5f 100644 --- a/.github/workflows/backend-selective.yml +++ b/.github/workflows/backend-selective.yml @@ -108,17 +108,20 @@ jobs: run: | set -euxo pipefail mkdir -p .coverage - gcloud storage cp "gs://sentry-coverage-data/${COVERAGE_SHA}/*" .coverage/ || true - # Find the coverage file (could be .coverage.* format) - COVERAGE_FILE=$(ls .coverage/.coverage.* 2>/dev/null | head -1 || true) - if [[ -z "$COVERAGE_FILE" ]]; then - echo "Warning: No coverage file found in downloaded data" + if ! gcloud storage cp "gs://sentry-coverage-data/${COVERAGE_SHA}/.coverage.combined" .coverage/; then + echo "Warning: Failed to download coverage file" + echo "coverage-file=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [[ ! -f .coverage/.coverage.combined ]]; then + echo "Warning: Coverage file not found after download" ls -la .coverage/ || true echo "coverage-file=" >> "$GITHUB_OUTPUT" else - echo "Downloaded coverage file: $COVERAGE_FILE" - echo "coverage-file=$COVERAGE_FILE" >> "$GITHUB_OUTPUT" + echo "Downloaded coverage file: .coverage/.coverage.combined" + echo "coverage-file=.coverage/.coverage.combined" >> "$GITHUB_OUTPUT" fi - name: Get changed files From 84a1c19d9a477ecf64952450f6d7745dd098805f Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Thu, 15 Jan 2026 09:42:06 -0800 Subject: [PATCH 26/42] Resiliency --- .github/workflows/backend-selective.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml index f5107b3cd01c5f..3c1ded70663f05 100644 --- a/.github/workflows/backend-selective.yml +++ b/.github/workflows/backend-selective.yml @@ -227,7 +227,9 @@ jobs: mode: backend-ci - name: Download coverage database artifact + id: download-coverage if: needs.prepare-selective-tests.outputs.has-coverage == 'true' + continue-on-error: true uses: actions/download-artifact@v4 with: name: coverage-db-${{ github.run_id }} @@ -235,10 +237,14 @@ jobs: - name: Find coverage file id: find-coverage-file - if: needs.prepare-selective-tests.outputs.has-coverage == 'true' run: | - COVERAGE_FILE=$(ls .coverage/.coverage.* 2>/dev/null | head -1 || true) - echo "coverage-file=$COVERAGE_FILE" >> "$GITHUB_OUTPUT" + if [[ -f .coverage/.coverage.combined ]]; then + echo "coverage-file=.coverage/.coverage.combined" >> "$GITHUB_OUTPUT" + echo "Coverage file found, selective testing enabled" + else + echo "coverage-file=" >> "$GITHUB_OUTPUT" + echo "No coverage file found, running full test suite" + fi - name: Run backend tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) id: run_backend_tests From 11c7547c4910884d7d6cecef18ce47a9be585420 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Thu, 15 Jan 2026 09:55:34 -0800 Subject: [PATCH 27/42] Makefile --- .github/workflows/backend-selective.yml | 11 +++++------ Makefile | 9 +++++++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml index 3c1ded70663f05..fc01d10520c6a3 100644 --- a/.github/workflows/backend-selective.yml +++ b/.github/workflows/backend-selective.yml @@ -138,12 +138,11 @@ jobs: - name: Compute selected tests id: compute-tests if: steps.download-coverage.outputs.coverage-file != '' - run: | - python3 .github/workflows/scripts/compute-selected-tests.py \ - --coverage-db "${{ steps.download-coverage.outputs.coverage-file }}" \ - --changed-files "${{ steps.changed-files.outputs.files }}" \ - --output .artifacts/selected-tests.txt \ - --github-output + env: + COVERAGE_DB: ${{ steps.download-coverage.outputs.coverage-file }} + CHANGED_FILES: ${{ steps.changed-files.outputs.files }} + IS_CI: true + run: make compute-selected-tests - name: Upload coverage database artifact if: steps.download-coverage.outputs.coverage-file != '' diff --git a/Makefile b/Makefile index 46dd5ef291ac49..081c898ed3cf7a 100644 --- a/Makefile +++ b/Makefile @@ -151,6 +151,15 @@ test-backend-ci-with-coverage: -o junit_suite_name=pytest @echo "" +compute-selected-tests: + @echo "--> Computing selected tests from coverage data" + python3 .github/workflows/scripts/compute-selected-tests.py \ + --coverage-db "$(COVERAGE_DB)" \ + --changed-files "$(CHANGED_FILES)" \ + --output .artifacts/selected-tests.txt \ + $(if $(IS_CI),--github-output,) + @echo "" + # it's not possible to change settings.DATABASE after django startup, so # unfortunately these tests must be run in a separate pytest process. References: # * https://docs.djangoproject.com/en/4.2/topics/testing/tools/#overriding-settings From 0bd2999843547102a2eb2e1d3ecd46e6785d5ec8 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Thu, 15 Jan 2026 21:33:07 -0800 Subject: [PATCH 28/42] Cleanups --- .github/workflows/backend-selective.yml | 28 +++---- .github/workflows/development-environment.yml | 71 ++++++++++++++++++ Makefile | 11 +++ .../testutils/pytest/selective_testing.py | 74 ------------------- src/sentry/testutils/pytest/sentry.py | 33 --------- 5 files changed, 91 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/development-environment.yml delete mode 100644 src/sentry/testutils/pytest/selective_testing.py diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml index fc01d10520c6a3..94e77eda5c6ef4 100644 --- a/.github/workflows/backend-selective.yml +++ b/.github/workflows/backend-selective.yml @@ -225,33 +225,23 @@ jobs: with: mode: backend-ci - - name: Download coverage database artifact - id: download-coverage - if: needs.prepare-selective-tests.outputs.has-coverage == 'true' + - name: Download selected tests artifact + if: needs.prepare-selective-tests.outputs.has-selected-tests == 'true' continue-on-error: true uses: actions/download-artifact@v4 with: - name: coverage-db-${{ github.run_id }} - path: .coverage/ + name: selected-tests-${{ github.run_id }} + path: .artifacts/ - - name: Find coverage file - id: find-coverage-file + - name: Run backend tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) + id: run_backend_tests run: | - if [[ -f .coverage/.coverage.combined ]]; then - echo "coverage-file=.coverage/.coverage.combined" >> "$GITHUB_OUTPUT" - echo "Coverage file found, selective testing enabled" + if [[ "${{ needs.prepare-selective-tests.outputs.has-selected-tests }}" == "true" && -f .artifacts/selected-tests.txt ]]; then + make test-python-ci-selective SELECTED_TESTS_FILE=.artifacts/selected-tests.txt else - echo "coverage-file=" >> "$GITHUB_OUTPUT" - echo "No coverage file found, running full test suite" + make test-python-ci fi - - name: Run backend tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) - id: run_backend_tests - run: make test-python-ci - env: - CHANGED_FILES: ${{ needs.prepare-selective-tests.outputs.changed-files }} - COVERAGE_DB_PATH: ${{ steps.find-coverage-file.outputs.coverage-file }} - - name: Inspect failure if: failure() run: | diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml new file mode 100644 index 00000000000000..42804f51630dda --- /dev/null +++ b/.github/workflows/development-environment.yml @@ -0,0 +1,71 @@ +name: dev env +on: + pull_request: + paths: + - '.pre-commit-config.yaml' + - 'Makefile' + - '.github/workflows/development-environment.yml' + - 'requirements-*.txt' + - 'pyproject.toml' + - 'uv.lock' + - '.python-version' + - '.envrc' + - 'Brewfile' + - 'scripts/**' + - 'tools/**' + - 'src/sentry/runner/commands/devserver.py' + - 'src/sentry/runner/commands/devservices.py' + - 'bin/load-mocks' + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 + +jobs: + test: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: astral-sh/setup-uv@884ad927a57e558e7a70b92f2bccf9198a4be546 # v6 + with: + version: '0.8.2' + # we just cache the venv-dir directly in action-setup-venv + enable-cache: false + + - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 + with: + cache-dependency-path: uv.lock + install-cmd: uv sync --only-dev --frozen --active + + - name: test-tools + run: make test-tools + + devenv: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: astral-sh/setup-uv@884ad927a57e558e7a70b92f2bccf9198a4be546 # v6 + with: + version: '0.8.2' + # we just cache the venv-dir directly in action-setup-venv + enable-cache: false + + - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 + with: + cache-dependency-path: uv.lock + # technically we can just use --only-dev but more cache is nice + install-cmd: uv sync --frozen --active + + - name: devenv sync + run: | + devenv --nocoderoot sync diff --git a/Makefile b/Makefile index 081c898ed3cf7a..bc90a697e3f99d 100644 --- a/Makefile +++ b/Makefile @@ -151,6 +151,17 @@ test-backend-ci-with-coverage: -o junit_suite_name=pytest @echo "" +test-backend-ci-selective: + @echo "--> Running CI Python tests (selective)" + python3 -b -m pytest \ + $$(cat $(SELECTED_TESTS_FILE)) \ + --json-report \ + --json-report-file=".artifacts/pytest.json" \ + --json-report-omit=log \ + --junit-xml=.artifacts/pytest.junit.xml \ + -o junit_suite_name=pytest + @echo "" + compute-selected-tests: @echo "--> Computing selected tests from coverage data" python3 .github/workflows/scripts/compute-selected-tests.py \ diff --git a/src/sentry/testutils/pytest/selective_testing.py b/src/sentry/testutils/pytest/selective_testing.py deleted file mode 100644 index f284d6232c7d19..00000000000000 --- a/src/sentry/testutils/pytest/selective_testing.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -import sqlite3 - -import pytest - -PYTEST_IGNORED_FILES = [ - # the pytest code itself is not part of the test suite but will be referenced by most tests - "sentry/testutils/pytest/sentry.py", -] - - -def filter_items_by_coverage( - config: pytest.Config, - items: list[pytest.Item], - changed_files: list[str], - coverage_db_path: str, -) -> tuple[list[pytest.Item], list[pytest.Item], set[str]]: - affected_test_files = set() - try: - conn = sqlite3.connect(coverage_db_path) - cur = conn.cursor() - - test_contexts = set() - - for file_path in changed_files: - if any(file_path.endswith(ignored_file) for ignored_file in PYTEST_IGNORED_FILES): - continue - - cleaned_file_path = file_path - if cleaned_file_path.startswith("/src"): - cleaned_file_path = cleaned_file_path[len("/src") :] - - cur.execute( - """ - SELECT c.context, lb.numbits - FROM line_bits lb - JOIN file f ON lb.file_id = f.id - JOIN context c ON lb.context_id = c.id - WHERE f.path LIKE '%' || ? - AND c.context != '' - """, - (f"%{cleaned_file_path}",), - ) - - for context, bitblob in cur.fetchall(): - if any(b != 0 for b in bytes(bitblob)): - test_contexts.add(context) - - conn.close() - - # Extract test file paths from contexts - # Context format: 'tests/foo/bar.py::TestClass::test_function|run' - for context in test_contexts: - test_file = context.split("::", 1)[0] - affected_test_files.add(test_file) - - except Exception as e: - raise ValueError(f"Could not query coverage database: {e}") from e - - config.get_terminal_writer().line(f"Found {len(affected_test_files)} affected test files") - config.get_terminal_writer().line(f"Affected test files: {affected_test_files}") - - selected_items = [] - discarded_items = [] - - for item in items: - test_file = item.nodeid.split("::", 1)[0] - if test_file in affected_test_files: - selected_items.append(item) - else: - discarded_items.append(item) - - return selected_items, discarded_items, affected_test_files diff --git a/src/sentry/testutils/pytest/sentry.py b/src/sentry/testutils/pytest/sentry.py index b9aaa86eb00758..547e8dfc0630f8 100644 --- a/src/sentry/testutils/pytest/sentry.py +++ b/src/sentry/testutils/pytest/sentry.py @@ -399,39 +399,6 @@ def _shuffle_d(dct: dict[K, V]) -> dict[K, V]: def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: """After collection, we need to select tests based on group and group strategy""" - # Selective test filtering based on coverage data - # If COVERAGE_DB_PATH and CHANGED_FILES are set, filter to tests that cover those files - coverage_db_path = os.environ.get("COVERAGE_DB_PATH") - changed_files_str = os.environ.get("CHANGED_FILES") - - if coverage_db_path and changed_files_str: - from sentry.testutils.pytest.selective_testing import filter_items_by_coverage - - changed_files = [f.strip() for f in changed_files_str.split() if f.strip()] - - if changed_files: - original_count = len(items) - try: - selected_items, deselected_items, _ = filter_items_by_coverage( - config=config, - items=items, - changed_files=changed_files, - coverage_db_path=coverage_db_path, - ) - - items[:] = selected_items - - if deselected_items: - config.hook.pytest_deselected(items=deselected_items) - - config.get_terminal_writer().line( - f"Selective testing: {len(items)}/{original_count} tests selected based on coverage" - ) - except ValueError as e: - config.get_terminal_writer().line( - f"Warning: Selective testing failed ({e}), running all tests" - ) - total_groups = int(os.environ.get("TOTAL_TEST_GROUPS", 1)) current_group = int(os.environ.get("TEST_GROUP", 0)) grouping_strategy = os.environ.get("TEST_GROUP_STRATEGY", "scope") From 5d7fdedc19dc22363a998b470f50f573c64a740a Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Thu, 15 Jan 2026 21:37:53 -0800 Subject: [PATCH 29/42] Rebase --- .github/workflows/backend-selective.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml index 94e77eda5c6ef4..38c57ac937b36e 100644 --- a/.github/workflows/backend-selective.yml +++ b/.github/workflows/backend-selective.yml @@ -237,7 +237,7 @@ jobs: id: run_backend_tests run: | if [[ "${{ needs.prepare-selective-tests.outputs.has-selected-tests }}" == "true" && -f .artifacts/selected-tests.txt ]]; then - make test-python-ci-selective SELECTED_TESTS_FILE=.artifacts/selected-tests.txt + make test-backend-ci-selective SELECTED_TESTS_FILE=.artifacts/selected-tests.txt else make test-python-ci fi From 8e7347d930b2571f2840e28d7e74a76f9ef98615 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Fri, 16 Jan 2026 10:44:31 -0800 Subject: [PATCH 30/42] Cleanup and force failure --- .github/workflows/backend-selective.yml | 5 ++--- .github/workflows/scripts/compute-selected-tests.py | 11 +---------- tests/sentry/preprod/size_analysis/test_compare.py | 2 +- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml index 38c57ac937b36e..64ade9eb63ed5d 100644 --- a/.github/workflows/backend-selective.yml +++ b/.github/workflows/backend-selective.yml @@ -201,10 +201,11 @@ jobs: name: backend test runs-on: ubuntu-24.04 timeout-minutes: 60 + continue-on-error: true permissions: contents: read id-token: write - actions: read + actions: read # used for DIM metadata strategy: fail-fast: false matrix: @@ -216,8 +217,6 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - fetch-depth: 0 - name: Setup sentry env uses: ./.github/actions/setup-sentry diff --git a/.github/workflows/scripts/compute-selected-tests.py b/.github/workflows/scripts/compute-selected-tests.py index 1a8b13bbe03cef..789e3be7a64615 100644 --- a/.github/workflows/scripts/compute-selected-tests.py +++ b/.github/workflows/scripts/compute-selected-tests.py @@ -1,10 +1,4 @@ #!/usr/bin/env python3 -""" -Compute selected tests based on coverage data and changed files. - -This script queries a coverage database to find which test files cover -the changed source files, outputting the list for selective test runs. -""" from __future__ import annotations import argparse @@ -13,13 +7,10 @@ import sys from pathlib import Path -PYTEST_IGNORED_FILES = [ - "sentry/testutils/pytest/sentry.py", -] +PYTEST_IGNORED_FILES = ["sentry/testutils/pytest/sentry.py", "pyproject.toml"] def get_affected_test_files(coverage_db_path: str, changed_files: list[str]) -> set[str]: - """Query coverage DB to find test files that cover the changed source files.""" affected_test_files: set[str] = set() conn = sqlite3.connect(coverage_db_path) diff --git a/tests/sentry/preprod/size_analysis/test_compare.py b/tests/sentry/preprod/size_analysis/test_compare.py index 469b823bae14f3..bda01b481ac998 100644 --- a/tests/sentry/preprod/size_analysis/test_compare.py +++ b/tests/sentry/preprod/size_analysis/test_compare.py @@ -114,7 +114,7 @@ def test_compare_size_analysis_file_added(self): metrics_artifact_type=PreprodArtifactSizeMetrics.MetricsArtifactType.MAIN_ARTIFACT, identifier="test", max_install_size=1500, - max_download_size=800, + max_download_size=900, ) # Head has one file, base has none From 6b5826f13925c34cdc275fda2ca38b94d9a1173c Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Fri, 16 Jan 2026 10:50:18 -0800 Subject: [PATCH 31/42] Force fail --- tests/sentry/preprod/size_analysis/test_compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sentry/preprod/size_analysis/test_compare.py b/tests/sentry/preprod/size_analysis/test_compare.py index bda01b481ac998..f9e9d500febc73 100644 --- a/tests/sentry/preprod/size_analysis/test_compare.py +++ b/tests/sentry/preprod/size_analysis/test_compare.py @@ -114,7 +114,7 @@ def test_compare_size_analysis_file_added(self): metrics_artifact_type=PreprodArtifactSizeMetrics.MetricsArtifactType.MAIN_ARTIFACT, identifier="test", max_install_size=1500, - max_download_size=900, + max_download_size=800, ) # Head has one file, base has none @@ -127,7 +127,7 @@ def test_compare_size_analysis_file_added(self): assert len(result.diff_items) == 1 diff_item = result.diff_items[0] assert diff_item.path == "file.txt" - assert diff_item.size_diff == 100 + assert diff_item.size_diff == 500 assert diff_item.head_size == 100 assert diff_item.base_size is None assert diff_item.type == DiffType.ADDED From ac0d587c5c34465c6b976b944c813891f6ca231d Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Fri, 16 Jan 2026 10:59:35 -0800 Subject: [PATCH 32/42] Cleanups and ready for review --- .github/workflows/backend-selective.yml | 4 ++-- src/sentry/preprod/size_analysis/compare.py | 2 -- tests/sentry/preprod/size_analysis/test_compare.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml index 64ade9eb63ed5d..094bbc586d2057 100644 --- a/.github/workflows/backend-selective.yml +++ b/.github/workflows/backend-selective.yml @@ -1,4 +1,4 @@ -name: backend (selective) +name: '[NOT REQUIRED] backend (selective)' on: pull_request: @@ -198,7 +198,7 @@ jobs: backend-test-selective: if: needs.files-changed.outputs.backend == 'true' needs: [files-changed, prepare-selective-tests, calculate-shards] - name: backend test + name: backend tests runs-on: ubuntu-24.04 timeout-minutes: 60 continue-on-error: true diff --git a/src/sentry/preprod/size_analysis/compare.py b/src/sentry/preprod/size_analysis/compare.py index 2996ca8c9d7cab..42d9b309236bdd 100644 --- a/src/sentry/preprod/size_analysis/compare.py +++ b/src/sentry/preprod/size_analysis/compare.py @@ -155,8 +155,6 @@ def compare_size_analysis( base_download_size=base_size_analysis.max_download_size, ) - # Placeholder - # Compare insights only if we're not skipping the comparison insight_diff_items = [] if not skip_diff_item_comparison: diff --git a/tests/sentry/preprod/size_analysis/test_compare.py b/tests/sentry/preprod/size_analysis/test_compare.py index f9e9d500febc73..469b823bae14f3 100644 --- a/tests/sentry/preprod/size_analysis/test_compare.py +++ b/tests/sentry/preprod/size_analysis/test_compare.py @@ -127,7 +127,7 @@ def test_compare_size_analysis_file_added(self): assert len(result.diff_items) == 1 diff_item = result.diff_items[0] assert diff_item.path == "file.txt" - assert diff_item.size_diff == 500 + assert diff_item.size_diff == 100 assert diff_item.head_size == 100 assert diff_item.base_size is None assert diff_item.type == DiffType.ADDED From 25b0fffa3989d0518d3a9b47a0501801008388b5 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Fri, 16 Jan 2026 11:16:54 -0800 Subject: [PATCH 33/42] Improvements --- .github/workflows/backend-selective.yml | 4 +++- .github/workflows/scripts/calculate-backend-test-shards.py | 1 - Makefile | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml index 094bbc586d2057..81d5c5d72cd845 100644 --- a/.github/workflows/backend-selective.yml +++ b/.github/workflows/backend-selective.yml @@ -19,6 +19,7 @@ jobs: name: detect what files changed runs-on: ubuntu-24.04 timeout-minutes: 3 + continue-on-error: true outputs: api_docs: ${{ steps.changes.outputs.api_docs }} backend: ${{ steps.changes.outputs.backend_all }} @@ -42,6 +43,7 @@ jobs: name: prepare selective tests runs-on: ubuntu-24.04 timeout-minutes: 10 + continue-on-error: true permissions: contents: read id-token: write @@ -141,7 +143,6 @@ jobs: env: COVERAGE_DB: ${{ steps.download-coverage.outputs.coverage-file }} CHANGED_FILES: ${{ steps.changed-files.outputs.files }} - IS_CI: true run: make compute-selected-tests - name: Upload coverage database artifact @@ -167,6 +168,7 @@ jobs: name: calculate test shards runs-on: ubuntu-24.04 timeout-minutes: 5 + continue-on-error: true outputs: shard-count: ${{ steps.calculate-shards.outputs.shard-count }} shard-indices: ${{ steps.calculate-shards.outputs.shard-indices }} diff --git a/.github/workflows/scripts/calculate-backend-test-shards.py b/.github/workflows/scripts/calculate-backend-test-shards.py index 13049589716796..071be270bbc78c 100755 --- a/.github/workflows/scripts/calculate-backend-test-shards.py +++ b/.github/workflows/scripts/calculate-backend-test-shards.py @@ -27,7 +27,6 @@ def collect_test_count(): - """Collect test count, optionally filtering to selected test files.""" selected_tests_file = os.environ.get("SELECTED_TESTS_FILE") if selected_tests_file: diff --git a/Makefile b/Makefile index bc90a697e3f99d..056738315dfaba 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ compute-selected-tests: --coverage-db "$(COVERAGE_DB)" \ --changed-files "$(CHANGED_FILES)" \ --output .artifacts/selected-tests.txt \ - $(if $(IS_CI),--github-output,) + --github-output @echo "" # it's not possible to change settings.DATABASE after django startup, so From f0093e30fc71c89e975fa2a0e54cda5a690e76e3 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Fri, 16 Jan 2026 11:39:44 -0800 Subject: [PATCH 34/42] Cleanups --- .github/workflows/scripts/calculate-backend-test-shards.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/scripts/calculate-backend-test-shards.py b/.github/workflows/scripts/calculate-backend-test-shards.py index 071be270bbc78c..cbee401184b501 100755 --- a/.github/workflows/scripts/calculate-backend-test-shards.py +++ b/.github/workflows/scripts/calculate-backend-test-shards.py @@ -16,9 +16,6 @@ "pytest", "--collect-only", "--quiet", -] - -PYTEST_IGNORE_ARGS = [ "--ignore=tests/acceptance", "--ignore=tests/apidocs", "--ignore=tests/js", @@ -45,7 +42,7 @@ def collect_test_count(): print(f"Counting tests in {len(selected_files)} selected files", file=sys.stderr) pytest_args = PYTEST_BASE_ARGS + selected_files else: - pytest_args = PYTEST_BASE_ARGS + ["tests"] + PYTEST_IGNORE_ARGS + pytest_args = PYTEST_BASE_ARGS + ["tests"] try: result = subprocess.run( From 54aa22cdc8de11b77920fef57726accb9e92e7d0 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Fri, 16 Jan 2026 11:49:42 -0800 Subject: [PATCH 35/42] Fix --- .github/workflows/backend-selective.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml index 81d5c5d72cd845..b5c1e1dac23bb2 100644 --- a/.github/workflows/backend-selective.yml +++ b/.github/workflows/backend-selective.yml @@ -228,7 +228,6 @@ jobs: - name: Download selected tests artifact if: needs.prepare-selective-tests.outputs.has-selected-tests == 'true' - continue-on-error: true uses: actions/download-artifact@v4 with: name: selected-tests-${{ github.run_id }} @@ -237,7 +236,7 @@ jobs: - name: Run backend tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) id: run_backend_tests run: | - if [[ "${{ needs.prepare-selective-tests.outputs.has-selected-tests }}" == "true" && -f .artifacts/selected-tests.txt ]]; then + if [[ "${{ needs.prepare-selective-tests.outputs.has-selected-tests }}" == "true" ]]; then make test-backend-ci-selective SELECTED_TESTS_FILE=.artifacts/selected-tests.txt else make test-python-ci From 32aac6ec3fbd22b6dc465078b32782a711bb7ded Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Fri, 16 Jan 2026 11:51:11 -0800 Subject: [PATCH 36/42] Add makefile to ignore --- .github/workflows/scripts/compute-selected-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts/compute-selected-tests.py b/.github/workflows/scripts/compute-selected-tests.py index 789e3be7a64615..366b0ccfcff042 100644 --- a/.github/workflows/scripts/compute-selected-tests.py +++ b/.github/workflows/scripts/compute-selected-tests.py @@ -7,7 +7,7 @@ import sys from pathlib import Path -PYTEST_IGNORED_FILES = ["sentry/testutils/pytest/sentry.py", "pyproject.toml"] +PYTEST_IGNORED_FILES = ["sentry/testutils/pytest/sentry.py", "pyproject.toml", "Makefile"] def get_affected_test_files(coverage_db_path: str, changed_files: list[str]) -> set[str]: From b257daef66f6db3511890834db5709cf3841f08c Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Fri, 16 Jan 2026 11:55:59 -0800 Subject: [PATCH 37/42] Fix full suite running --- .../scripts/compute-selected-tests.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/scripts/compute-selected-tests.py b/.github/workflows/scripts/compute-selected-tests.py index 366b0ccfcff042..8ca297d166fbaf 100644 --- a/.github/workflows/scripts/compute-selected-tests.py +++ b/.github/workflows/scripts/compute-selected-tests.py @@ -7,7 +7,15 @@ import sys from pathlib import Path -PYTEST_IGNORED_FILES = ["sentry/testutils/pytest/sentry.py", "pyproject.toml", "Makefile"] +# Files that, if changed, should trigger the full test suite (can't determine affected tests) +FULL_SUITE_TRIGGER_FILES = ["sentry/testutils/pytest/sentry.py", "pyproject.toml", "Makefile"] + + +def should_run_full_suite(changed_files: list[str]) -> bool: + for file_path in changed_files: + if any(file_path.endswith(trigger) for trigger in FULL_SUITE_TRIGGER_FILES): + return True + return False def get_affected_test_files(coverage_db_path: str, changed_files: list[str]) -> set[str]: @@ -29,9 +37,6 @@ def get_affected_test_files(coverage_db_path: str, changed_files: list[str]) -> test_contexts: set[str] = set() for file_path in changed_files: - if any(file_path.endswith(ignored_file) for ignored_file in PYTEST_IGNORED_FILES): - continue - cleaned_file_path = file_path if cleaned_file_path.startswith("/src"): cleaned_file_path = cleaned_file_path[len("/src") :] @@ -80,8 +85,14 @@ def main() -> int: changed_files = [f.strip() for f in args.changed_files.split() if f.strip()] if not changed_files: - print("No changed files provided, selecting all tests") + print("No changed files provided, running full test suite") affected_test_files: set[str] = set() + elif should_run_full_suite(changed_files): + triggered_by = [ + f for f in changed_files if any(f.endswith(t) for t in FULL_SUITE_TRIGGER_FILES) + ] + print(f"Full test suite triggered by: {', '.join(triggered_by)}") + affected_test_files = set() else: print(f"Computing selected tests for {len(changed_files)} changed files...") try: From 52d3c99be35f1b30edd11f4e523ba2b458efdc8c Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Tue, 20 Jan 2026 08:23:15 -0800 Subject: [PATCH 38/42] Tweaks --- .../scripts/calculate-backend-test-shards.py | 7 +++-- .../scripts/compute-selected-tests.py | 28 +++++++++++++++---- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/scripts/calculate-backend-test-shards.py b/.github/workflows/scripts/calculate-backend-test-shards.py index cbee401184b501..e2677653ea7197 100755 --- a/.github/workflows/scripts/calculate-backend-test-shards.py +++ b/.github/workflows/scripts/calculate-backend-test-shards.py @@ -23,7 +23,8 @@ ] -def collect_test_count(): +def collect_test_count() -> int | None: + """Collect the number of tests to run, either from selected files or full suite.""" selected_tests_file = os.environ.get("SELECTED_TESTS_FILE") if selected_tests_file: @@ -75,7 +76,7 @@ def collect_test_count(): return None -def calculate_shards(test_count): +def calculate_shards(test_count: int | None) -> int: if test_count is None: print(f"Using default shard count: {DEFAULT_SHARDS}", file=sys.stderr) return DEFAULT_SHARDS @@ -101,7 +102,7 @@ def calculate_shards(test_count): return bounded -def main(): +def main() -> int: test_count = collect_test_count() shard_count = calculate_shards(test_count) shard_indices = json.dumps(list(range(shard_count))) diff --git a/.github/workflows/scripts/compute-selected-tests.py b/.github/workflows/scripts/compute-selected-tests.py index 8ca297d166fbaf..c044f658f5e9c9 100644 --- a/.github/workflows/scripts/compute-selected-tests.py +++ b/.github/workflows/scripts/compute-selected-tests.py @@ -8,7 +8,13 @@ from pathlib import Path # Files that, if changed, should trigger the full test suite (can't determine affected tests) -FULL_SUITE_TRIGGER_FILES = ["sentry/testutils/pytest/sentry.py", "pyproject.toml", "Makefile"] +FULL_SUITE_TRIGGER_FILES = [ + "sentry/testutils/pytest/sentry.py", + "pyproject.toml", + "Makefile", + "sentry/conf/server.py", + "sentry/web/urls.py", +] def should_run_full_suite(changed_files: list[str]) -> bool: @@ -18,6 +24,15 @@ def should_run_full_suite(changed_files: list[str]) -> bool: return False +def get_changed_test_files(changed_files: list[str]) -> set[str]: + test_files: set[str] = set() + for file_path in changed_files: + # Match test files in the tests/ directory + if file_path.startswith("tests/") and file_path.endswith(".py"): + test_files.add(file_path) + return test_files + + def get_affected_test_files(coverage_db_path: str, changed_files: list[str]) -> set[str]: affected_test_files: set[str] = set() @@ -37,9 +52,6 @@ def get_affected_test_files(coverage_db_path: str, changed_files: list[str]) -> test_contexts: set[str] = set() for file_path in changed_files: - cleaned_file_path = file_path - if cleaned_file_path.startswith("/src"): - cleaned_file_path = cleaned_file_path[len("/src") :] cur.execute( """ @@ -50,7 +62,7 @@ def get_affected_test_files(coverage_db_path: str, changed_files: list[str]) -> WHERE f.path LIKE '%' || ? AND c.context != '' """, - (f"%{cleaned_file_path}",), + (f"%{file_path}",), ) for context, bitblob in cur.fetchall(): @@ -101,6 +113,12 @@ def main() -> int: print(f"Error querying coverage database: {e}", file=sys.stderr) return 1 + # Also include any test files that were directly changed/added in the PR + changed_test_files = get_changed_test_files(changed_files) + if changed_test_files: + print(f"Including {len(changed_test_files)} directly changed test files") + affected_test_files.update(changed_test_files) + print(f"Found {len(affected_test_files)} affected test files") if args.output: From 043cfd8bfb8e652c330554634289f9fb5add71f3 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Tue, 20 Jan 2026 09:27:20 -0800 Subject: [PATCH 39/42] Sanity check/debug --- .github/workflows/scripts/compute-selected-tests.py | 3 ++- src/sentry/preprod/size_analysis/compare.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts/compute-selected-tests.py b/.github/workflows/scripts/compute-selected-tests.py index c044f658f5e9c9..97ad38db29262f 100644 --- a/.github/workflows/scripts/compute-selected-tests.py +++ b/.github/workflows/scripts/compute-selected-tests.py @@ -11,7 +11,7 @@ FULL_SUITE_TRIGGER_FILES = [ "sentry/testutils/pytest/sentry.py", "pyproject.toml", - "Makefile", + # "Makefile", "sentry/conf/server.py", "sentry/web/urls.py", ] @@ -52,6 +52,7 @@ def get_affected_test_files(coverage_db_path: str, changed_files: list[str]) -> test_contexts: set[str] = set() for file_path in changed_files: + print(f"Processing file: {file_path}") cur.execute( """ diff --git a/src/sentry/preprod/size_analysis/compare.py b/src/sentry/preprod/size_analysis/compare.py index 42d9b309236bdd..f92080247616e8 100644 --- a/src/sentry/preprod/size_analysis/compare.py +++ b/src/sentry/preprod/size_analysis/compare.py @@ -29,6 +29,7 @@ logger = logging.getLogger(__name__) +# Placeholder def compare_size_analysis( head_size_analysis: PreprodArtifactSizeMetrics, head_size_analysis_results: SizeAnalysisResults, From e3e0501ba55bd61671faf6c0217fc760c956876e Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Tue, 20 Jan 2026 10:00:31 -0800 Subject: [PATCH 40/42] Revert debugging --- .github/workflows/scripts/compute-selected-tests.py | 2 -- src/sentry/preprod/size_analysis/compare.py | 1 - 2 files changed, 3 deletions(-) diff --git a/.github/workflows/scripts/compute-selected-tests.py b/.github/workflows/scripts/compute-selected-tests.py index 97ad38db29262f..50eeab12da8205 100644 --- a/.github/workflows/scripts/compute-selected-tests.py +++ b/.github/workflows/scripts/compute-selected-tests.py @@ -52,8 +52,6 @@ def get_affected_test_files(coverage_db_path: str, changed_files: list[str]) -> test_contexts: set[str] = set() for file_path in changed_files: - print(f"Processing file: {file_path}") - cur.execute( """ SELECT c.context, lb.numbits diff --git a/src/sentry/preprod/size_analysis/compare.py b/src/sentry/preprod/size_analysis/compare.py index f92080247616e8..42d9b309236bdd 100644 --- a/src/sentry/preprod/size_analysis/compare.py +++ b/src/sentry/preprod/size_analysis/compare.py @@ -29,7 +29,6 @@ logger = logging.getLogger(__name__) -# Placeholder def compare_size_analysis( head_size_analysis: PreprodArtifactSizeMetrics, head_size_analysis_results: SizeAnalysisResults, From 19db7ab38b1c0a366fe5dfa74c4007688e97d3b1 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Tue, 20 Jan 2026 10:05:02 -0800 Subject: [PATCH 41/42] Adjust --- .github/workflows/scripts/compute-selected-tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/scripts/compute-selected-tests.py b/.github/workflows/scripts/compute-selected-tests.py index 50eeab12da8205..4e2c4cde4305cb 100644 --- a/.github/workflows/scripts/compute-selected-tests.py +++ b/.github/workflows/scripts/compute-selected-tests.py @@ -112,11 +112,11 @@ def main() -> int: print(f"Error querying coverage database: {e}", file=sys.stderr) return 1 - # Also include any test files that were directly changed/added in the PR - changed_test_files = get_changed_test_files(changed_files) - if changed_test_files: - print(f"Including {len(changed_test_files)} directly changed test files") - affected_test_files.update(changed_test_files) + # Also include any test files that were directly changed/added in the PR + changed_test_files = get_changed_test_files(changed_files) + if changed_test_files: + print(f"Including {len(changed_test_files)} directly changed test files") + affected_test_files.update(changed_test_files) print(f"Found {len(affected_test_files)} affected test files") From c530b88947c70935811cc20e32ae3324f2b820f8 Mon Sep 17 00:00:00 2001 From: Ryan Brooks Date: Tue, 20 Jan 2026 10:20:25 -0800 Subject: [PATCH 42/42] Preprod --- .github/workflows/backend-selective.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backend-selective.yml b/.github/workflows/backend-selective.yml index b5c1e1dac23bb2..96e9437646c040 100644 --- a/.github/workflows/backend-selective.yml +++ b/.github/workflows/backend-selective.yml @@ -2,9 +2,9 @@ name: '[NOT REQUIRED] backend (selective)' on: pull_request: + paths: + - 'src/sentry/preprod/**' -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true