From 6f2aa2cafce031fcaf25590cd3b41d003e4406a0 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 10:39:02 -0700 Subject: [PATCH 01/13] ref(ci): overlap devservices startup with venv setup in backend tests setup-sentry takes ~2min per backend test shard, with ~90s spent on devservices (Docker image pulls + container health checks). This all runs serially after Python/venv setup, even though image pulls don't depend on the venv at all. Add a setup-devservices composite action that: - Parses uv.lock (via tomllib) for the pinned devservices version - Installs it into an isolated /tmp venv using system Python - Starts `devservices up` in the background immediately after checkout The full setup-sentry venv restore (~19s) now runs concurrently with Docker image pulls. A wait script polls for completion with proper error handling and timeout. Also extract the Snuba per-worker bootstrap into a two-phase script: - Phase 1 (early): Polls for ClickHouse readiness and runs the expensive `bootstrap --force` while devservices is still bringing up remaining containers - Phase 2 (after devservices): Stops the default Snuba container and starts per-worker API instances Net saving: ~25-35s per shard depending on container startup order. --- .github/actions/setup-devservices/action.yml | 38 ++++ .../setup-devservices/bootstrap-snuba.sh | 108 ++++++++++ .github/actions/setup-devservices/wait.sh | 26 +++ .github/workflows/backend-with-coverage.yml | 199 ------------------ .github/workflows/backend.yml | 75 +++---- 5 files changed, 199 insertions(+), 247 deletions(-) create mode 100644 .github/actions/setup-devservices/action.yml create mode 100755 .github/actions/setup-devservices/bootstrap-snuba.sh create mode 100755 .github/actions/setup-devservices/wait.sh delete mode 100644 .github/workflows/backend-with-coverage.yml diff --git a/.github/actions/setup-devservices/action.yml b/.github/actions/setup-devservices/action.yml new file mode 100644 index 00000000000000..496a7da5cdb717 --- /dev/null +++ b/.github/actions/setup-devservices/action.yml @@ -0,0 +1,38 @@ +name: 'Early Devservices' +description: 'Starts devservices in the background so image pulls overlap with venv setup' +inputs: + mode: + description: 'devservices mode (must match the mode passed to setup-sentry)' + required: true + timeout-minutes: + description: 'Maximum minutes for devservices up' + required: false + default: '10' + +runs: + using: 'composite' + steps: + - uses: astral-sh/setup-uv@884ad927a57e558e7a70b92f2bccf9198a4be546 # v6 + with: + version: '0.9.28' + enable-cache: false + + - name: Start devservices in background + shell: bash + run: | + DS_VERSION=$(python3 -c " + import tomllib + with open('uv.lock', 'rb') as f: + lock = tomllib.load(f) + for pkg in lock['package']: + if pkg['name'] == 'devservices': + print(pkg['version']) + break + ") + echo "Installing devservices==${DS_VERSION}" + uv venv /tmp/ds-venv --python python3 -q + uv pip install --python /tmp/ds-venv/bin/python -q \ + --index-url https://pypi.devinfra.sentry.io/simple \ + "devservices==${DS_VERSION}" + (timeout ${{ inputs.timeout-minutes }}m /tmp/ds-venv/bin/devservices up --mode ${{ inputs.mode }}; echo $? > /tmp/ds-exit) \ + > /tmp/ds.log 2>&1 & diff --git a/.github/actions/setup-devservices/bootstrap-snuba.sh b/.github/actions/setup-devservices/bootstrap-snuba.sh new file mode 100755 index 00000000000000..82e771505772c3 --- /dev/null +++ b/.github/actions/setup-devservices/bootstrap-snuba.sh @@ -0,0 +1,108 @@ +#!/bin/bash +set -euo pipefail + +# Bootstrap per-worker Snuba instances, overlapping the expensive ClickHouse +# table setup with the devservices health-check wait. +# +# Phase 1 (early): As soon as ClickHouse is accepting queries, create per-worker +# databases and run `snuba bootstrap --force`. This is the slow part. +# Phase 2 (after devservices): Stop snuba-snuba-1 and start per-worker API +# containers. We must wait for devservices to finish first — stopping the +# container while devservices is health-checking it would cause a timeout. +# +# Requires: XDIST_WORKERS env var +# Reads: /tmp/ds-exit (written by setup-devservices/wait.sh) +# Writes: /tmp/snuba-bootstrap-exit + +WORKERS=${XDIST_WORKERS:?XDIST_WORKERS must be set} + +echo "Waiting for ClickHouse and Snuba container..." +SECONDS=0 +while true; do + if [ $SECONDS -gt 300 ]; then + echo "::error::Timed out waiting for Snuba bootstrap prerequisites" + echo 1 > /tmp/snuba-bootstrap-exit + exit 1 + fi + if curl -sf 'http://localhost:8123/' > /dev/null 2>&1 \ + && docker inspect snuba-snuba-1 > /dev/null 2>&1; then + break + fi + sleep 2 +done +echo "Prerequisites ready (${SECONDS}s)" + +SNUBA_IMAGE=$(docker inspect snuba-snuba-1 --format '{{.Config.Image}}') +SNUBA_NETWORK=$(docker inspect snuba-snuba-1 --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}') +if [ -z "$SNUBA_IMAGE" ] || [ -z "$SNUBA_NETWORK" ]; then + echo "::error::Could not inspect snuba-snuba-1 container" + echo 1 > /tmp/snuba-bootstrap-exit + exit 1 +fi + +SNUBA_ENV=( + -e "CLICKHOUSE_HOST=clickhouse" -e "CLICKHOUSE_PORT=9000" -e "CLICKHOUSE_HTTP_PORT=8123" + -e "DEFAULT_BROKERS=kafka:9093" -e "REDIS_HOST=redis" -e "REDIS_PORT=6379" -e "REDIS_DB=1" + -e "SNUBA_SETTINGS=docker" +) + +# Phase 1: Create databases and run bootstrap (the expensive part). +# This can safely run while devservices is still health-checking containers. +echo "Phase 1: bootstrapping ClickHouse databases" +BOOTSTRAP_PIDS=() +for i in $(seq 0 $(( WORKERS - 1 ))); do + ( + WORKER_DB="default_gw${i}" + curl -sf 'http://localhost:8123/' --data-binary "CREATE DATABASE IF NOT EXISTS ${WORKER_DB}" + docker run --rm --network "$SNUBA_NETWORK" \ + -e "CLICKHOUSE_DATABASE=${WORKER_DB}" "${SNUBA_ENV[@]}" \ + "$SNUBA_IMAGE" bootstrap --force 2>&1 | tail -3 + ) & + BOOTSTRAP_PIDS+=($!) +done + +for pid in "${BOOTSTRAP_PIDS[@]}"; do + wait "$pid" || { echo "ERROR: Snuba bootstrap (PID $pid) failed"; echo 1 > /tmp/snuba-bootstrap-exit; exit 1; } +done +echo "Phase 1 done (${SECONDS}s)" + +# Phase 2: Wait for devservices to finish, then swap snuba-snuba-1 for per-worker containers. +while [ ! -f /tmp/ds-exit ]; do sleep 1; done + +docker stop snuba-snuba-1 || true + +echo "Phase 2: starting per-worker Snuba API containers" +GW_PIDS=() +for i in $(seq 0 $(( WORKERS - 1 ))); do + ( + WORKER_DB="default_gw${i}" + WORKER_PORT=$((1230 + i)) + docker run -d --name "snuba-gw${i}" --network "$SNUBA_NETWORK" \ + -p "${WORKER_PORT}:1218" \ + -e "CLICKHOUSE_DATABASE=${WORKER_DB}" "${SNUBA_ENV[@]}" \ + -e "DEBUG=1" "$SNUBA_IMAGE" api + + for attempt in $(seq 1 30); do + if curl -sf "http://127.0.0.1:${WORKER_PORT}/health" > /dev/null 2>&1; then + echo "snuba-gw${i} healthy on port ${WORKER_PORT}" + break + fi + if [ "$attempt" -eq 30 ]; then + echo "ERROR: snuba-gw${i} failed health check after 30 attempts" + docker logs "snuba-gw${i}" 2>&1 | tail -20 || true + exit 1 + fi + sleep 2 + done + ) & + GW_PIDS+=($!) +done + +RC=0 +for pid in "${GW_PIDS[@]}"; do + wait "$pid" || { echo "ERROR: Snuba gateway (PID $pid) failed"; RC=1; } +done + +echo "Snuba bootstrap complete (${SECONDS}s total)" +echo $RC > /tmp/snuba-bootstrap-exit +exit $RC diff --git a/.github/actions/setup-devservices/wait.sh b/.github/actions/setup-devservices/wait.sh new file mode 100755 index 00000000000000..6f07708be84407 --- /dev/null +++ b/.github/actions/setup-devservices/wait.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +# Wait for the background devservices process started by the setup-devservices action. +# Usage: wait.sh [timeout_seconds] +TIMEOUT=${1:-600} + +SECONDS=0 +while [ ! -f /tmp/ds-exit ]; do + if [ $SECONDS -gt "$TIMEOUT" ]; then + echo "::error::Timed out waiting for devservices after ${TIMEOUT}s" + cat /tmp/ds.log + exit 1 + fi + sleep 2 +done + +DS_RC=$(cat /tmp/ds-exit) +if [ "$DS_RC" -ne 0 ]; then + echo "::error::devservices up failed (exit $DS_RC)" + cat /tmp/ds.log + exit 1 +fi + +echo "DJANGO_LIVE_TEST_SERVER_ADDRESS=$(docker network inspect bridge --format='{{(index .IPAM.Config 0).Gateway}}')" >> "$GITHUB_ENV" +docker ps -a diff --git a/.github/workflows/backend-with-coverage.yml b/.github/workflows/backend-with-coverage.yml deleted file mode 100644 index 65f1496ce76571..00000000000000 --- a/.github/workflows/backend-with-coverage.yml +++ /dev/null @@ -1,199 +0,0 @@ -name: backend - with test coverage - -on: - workflow_dispatch: - workflow_call: - # Every 30 minutes - schedule: - - cron: '0,30 * * * *' - -env: - SENTRY_SKIP_SELENIUM_PLUGIN: '1' - -jobs: - # Skip generating coverage if already exists in GCS - check-existing-coverage: - name: check for existing coverage - runs-on: ubuntu-24.04 - timeout-minutes: 5 - permissions: - contents: read - id-token: write - outputs: - has-coverage: ${{ steps.check-coverage.outputs.exists }} - commit-sha: ${{ steps.get-sha.outputs.sha }} - steps: - - name: Determine commit SHA - id: get-sha - run: | - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - COMMIT_SHA="${{ github.event.pull_request.head.sha }}" - else - COMMIT_SHA="${{ github.sha }}" - fi - echo "sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2.1.3 - with: - project_id: sentry-dev-tooling - workload_identity_provider: ${{ secrets.SENTRY_GCP_DEV_WORKLOAD_IDENTITY_POOL }} - service_account: ${{ secrets.COLLECT_TEST_DATA_SERVICE_ACCOUNT_EMAIL }} - - - name: Check if coverage exists - id: check-coverage - env: - COMMIT_SHA: ${{ steps.get-sha.outputs.sha }} - run: | - if gcloud storage ls "gs://sentry-coverage-data/${COMMIT_SHA}/.coverage.combined" &>/dev/null; then - echo "Coverage already exists for commit ${COMMIT_SHA}, skipping test run" - echo "exists=true" >> "$GITHUB_OUTPUT" - else - echo "No coverage found for commit ${COMMIT_SHA}, will run tests" - echo "exists=false" >> "$GITHUB_OUTPUT" - fi - - calculate-shards: - name: calculate test shards - if: needs.check-existing-coverage.outputs.has-coverage != 'true' - needs: [check-existing-coverage] - runs-on: ubuntu-24.04 - timeout-minutes: 5 - outputs: - shard-count: ${{ steps.calculate-shards.outputs.shard-count }} - shard-indices: ${{ steps.calculate-shards.outputs.shard-indices }} - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - mode: backend-ci - skip-devservices: true - - - name: Calculate test shards - id: calculate-shards - run: python3 .github/workflows/scripts/calculate-backend-test-shards.py - - backend-test-with-cov-context: - name: backend test - if: needs.check-existing-coverage.outputs.has-coverage != 'true' - runs-on: ubuntu-24.04 - needs: [check-existing-coverage, calculate-shards] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - instance: ${{ fromJSON(needs.calculate-shards.outputs.shard-indices) }} - env: - MATRIX_INSTANCE_TOTAL: ${{ needs.calculate-shards.outputs.shard-count }} - TEST_GROUP_STRATEGY: roundrobin - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - mode: backend-ci - - - name: Download odiff binary - run: | - curl -sL https://registry.npmjs.org/odiff-bin/-/odiff-bin-4.3.2.tgz \ - | tar -xz --strip-components=2 package/raw_binaries/odiff-linux-x64 - sudo install -m 755 odiff-linux-x64 /usr/local/bin/odiff - rm odiff-linux-x64 - - - name: Run backend test with coverage (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) - run: make test-backend-ci-with-coverage - - - name: Validate coverage database - if: always() - run: | - set -euxo pipefail - if [[ ! -f .coverage ]]; then - echo "Error: No .coverage file found after tests" - exit 1 - fi - python -c "import sqlite3; sqlite3.connect('.coverage').execute('SELECT COUNT(*) FROM file')" - - - name: Upload raw coverage sqlite as artifact - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: pycoverage-sqlite-${{ github.run_id }}-${{ steps.setup.outputs.matrix-instance-number }} - path: .coverage - if-no-files-found: error - retention-days: 7 - include-hidden-files: true - - combine-coverage: - name: combine coverage - # Only upload coverage if all test shards pass - incomplete coverage could cause selective testing to skip tests incorrectly - if: needs.check-existing-coverage.outputs.has-coverage != 'true' - runs-on: ubuntu-24.04 - permissions: - contents: read - id-token: write - actions: read # used for DIM metadata - needs: [check-existing-coverage, backend-test-with-cov-context, calculate-shards] - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - uses: astral-sh/setup-uv@884ad927a57e558e7a70b92f2bccf9198a4be546 # v6 - with: - version: '0.9.28' - enable-cache: false - - - name: Download all coverage artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - pattern: pycoverage-sqlite-${{ github.run_id }}-* - path: .artifacts/all-coverage - - - name: Verify all shards produced coverage - env: - EXPECTED_SHARD_COUNT: ${{ needs.calculate-shards.outputs.shard-count }} - run: | - set -euxo pipefail - - echo "Downloaded coverage artifacts:" - ls -la .artifacts/all-coverage || true - - COVERAGE_FILE_COUNT=$(find .artifacts/all-coverage -name ".coverage" -type f | wc -l) - echo "Found ${COVERAGE_FILE_COUNT} coverage files, expected ${EXPECTED_SHARD_COUNT}" - - if [[ "$COVERAGE_FILE_COUNT" -ne "$EXPECTED_SHARD_COUNT" ]]; then - echo "Error: Missing coverage files. Expected ${EXPECTED_SHARD_COUNT}, found ${COVERAGE_FILE_COUNT}" - echo "This indicates some test shards failed to produce coverage." - find .artifacts/all-coverage -name ".coverage" -type f - exit 1 - fi - - - name: Combine all coverage databases - run: | - set -euxo pipefail - uvx --with covdefaults --with sentry-covdefaults-disable-branch-coverage \ - coverage combine $(find .artifacts/all-coverage -name ".coverage" -type f) - - if [[ ! -f .coverage ]]; then - echo "Error: Combined coverage file was not created" - exit 1 - fi - - mv .coverage ".coverage.combined" - - - name: Authenticate to Google Cloud - id: gcloud-auth - uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2.1.3 - with: - project_id: sentry-dev-tooling - workload_identity_provider: ${{ secrets.SENTRY_GCP_DEV_WORKLOAD_IDENTITY_POOL }} - service_account: ${{ secrets.COLLECT_TEST_DATA_SERVICE_ACCOUNT_EMAIL }} - - - name: Upload coverage to GCS - uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2.2.4 - with: - path: .coverage.combined - destination: sentry-coverage-data/${{ needs.check-existing-coverage.outputs.commit-sha }} diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 23b62cfad4243c..ab98d271258551 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -234,11 +234,28 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Start devservices early + uses: ./.github/actions/setup-devservices + with: + mode: backend-ci + - name: Setup sentry env uses: ./.github/actions/setup-sentry id: setup with: mode: backend-ci + skip-devservices: 'true' + + - name: Wait for devservices + run: | + sentry init + # Start Snuba per-worker bootstrap in background — it polls for + # ClickHouse readiness itself, so it can begin before all containers + # are healthy, overlapping with the remaining devservices health checks. + if [ "${XDIST_PER_WORKER_SNUBA}" = "1" ]; then + ./.github/actions/setup-devservices/bootstrap-snuba.sh & + fi + ./.github/actions/setup-devservices/wait.sh - name: Download odiff binary run: | @@ -247,59 +264,16 @@ jobs: sudo install -m 755 odiff-linux-x64 /usr/local/bin/odiff rm odiff-linux-x64 - - name: Bootstrap per-worker Snuba instances + - name: Wait for Snuba bootstrap if: env.XDIST_PER_WORKER_SNUBA == '1' run: | - set -eo pipefail - SNUBA_IMAGE=$(docker inspect snuba-snuba-1 --format '{{.Config.Image}}') - SNUBA_NETWORK=$(docker inspect snuba-snuba-1 --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}') - if [ -z "$SNUBA_IMAGE" ] || [ -z "$SNUBA_NETWORK" ]; then - echo "ERROR: Could not inspect snuba-snuba-1 container. Is devservices running?" + while [ ! -f /tmp/snuba-bootstrap-exit ]; do sleep 2; done + SNUBA_RC=$(cat /tmp/snuba-bootstrap-exit) + if [ "$SNUBA_RC" -ne 0 ]; then + echo "::error::Snuba per-worker bootstrap failed" exit 1 fi - docker stop snuba-snuba-1 || true - - PIDS=() - for i in $(seq 0 $(( ${XDIST_WORKERS} - 1 ))); do - ( - WORKER_DB="default_gw${i}" - WORKER_PORT=$((1230 + i)) - curl -sf 'http://localhost:8123/' --data-binary "CREATE DATABASE IF NOT EXISTS ${WORKER_DB}" - docker run --rm --network "$SNUBA_NETWORK" \ - -e "CLICKHOUSE_DATABASE=${WORKER_DB}" -e "CLICKHOUSE_HOST=clickhouse" \ - -e "CLICKHOUSE_PORT=9000" -e "CLICKHOUSE_HTTP_PORT=8123" \ - -e "DEFAULT_BROKERS=kafka:9093" -e "REDIS_HOST=redis" \ - -e "REDIS_PORT=6379" -e "REDIS_DB=1" -e "SNUBA_SETTINGS=docker" \ - "$SNUBA_IMAGE" bootstrap --force 2>&1 | tail -3 - docker run -d --name "snuba-gw${i}" --network "$SNUBA_NETWORK" \ - -p "${WORKER_PORT}:1218" \ - -e "CLICKHOUSE_DATABASE=${WORKER_DB}" -e "CLICKHOUSE_HOST=clickhouse" \ - -e "CLICKHOUSE_PORT=9000" -e "CLICKHOUSE_HTTP_PORT=8123" \ - -e "DEFAULT_BROKERS=kafka:9093" -e "REDIS_HOST=redis" \ - -e "REDIS_PORT=6379" -e "REDIS_DB=1" -e "SNUBA_SETTINGS=docker" \ - -e "DEBUG=1" "$SNUBA_IMAGE" api - - for attempt in $(seq 1 30); do - if curl -sf "http://127.0.0.1:${WORKER_PORT}/health" > /dev/null 2>&1; then - echo "snuba-gw${i} healthy on port ${WORKER_PORT}" - break - fi - if [ "$attempt" -eq 30 ]; then - echo "ERROR: snuba-gw${i} failed health check after 30 attempts" - docker logs "snuba-gw${i}" 2>&1 | tail -20 || true - exit 1 - fi - sleep 2 - done - ) & - PIDS+=($!) - done - - for pid in "${PIDS[@]}"; do - wait "$pid" || { echo "ERROR: Snuba bootstrap subshell (PID $pid) failed"; exit 1; } - done - - name: Download selected tests artifact if: needs.select-tests.outputs.has-selected-tests == 'true' uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 @@ -339,6 +313,11 @@ jobs: - name: Inspect failure if: failure() run: | + if [ -f /tmp/ds.log ]; then + echo "--- devservices startup log ---" + cat /tmp/ds.log + fi + if command -v devservices; then devservices logs fi From e8287ab26e04b9e4936ce474250986c83810d5be Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 10:40:18 -0700 Subject: [PATCH 02/13] ref(ci): overlap devservices with webpack build in acceptance tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply the same early-devservices pattern from the backend test change. Acceptance tests have several minutes of webpack/node build between checkout and setup-sentry, so devservices will be fully ready by the time the venv is set up — the entire ~90s of container startup becomes free. --- .github/workflows/acceptance.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 8bad8cfafc0683..dbe9d9edf2f0e8 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -68,6 +68,11 @@ jobs: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 name: Checkout sentry + - name: Start devservices early + uses: ./.github/actions/setup-devservices + with: + mode: acceptance-ci + - name: Step configurations id: config run: | @@ -98,6 +103,12 @@ jobs: id: setup with: mode: acceptance-ci + skip-devservices: 'true' + + - name: Wait for devservices + run: | + sentry init + ./.github/actions/setup-devservices/wait.sh - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) run: make run-acceptance @@ -105,6 +116,11 @@ jobs: - name: Inspect failure if: failure() run: | + if [ -f /tmp/ds.log ]; then + echo "--- devservices startup log ---" + cat /tmp/ds.log + fi + if command -v devservices; then devservices logs fi From aa24e0334bd0fba64f03ce3346105d4f898decac Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 10:48:07 -0700 Subject: [PATCH 03/13] ref(ci): apply early devservices to all remaining setup-sentry callers Apply the same overlap pattern to every workflow that uses setup-sentry: backend-migration-tests, cli, api-docs, api-url-typescript, migration, monolith-dbs, migrations, migrations-drift, openapi, and openapi-diff. --- .github/workflows/backend.yml | 67 ++++++++++++++++++++++++++ .github/workflows/migrations-drift.yml | 11 +++++ .github/workflows/migrations.yml | 11 +++++ .github/workflows/openapi-diff.yml | 13 +++++ .github/workflows/openapi.yml | 13 +++++ 5 files changed, 115 insertions(+) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index ab98d271258551..330baf5fe5091d 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -59,11 +59,23 @@ jobs: node-version-file: '.node-version' - uses: pnpm/action-setup@9b5745cdf0a2e8c2620f0746130f809adb911c19 # v4 + + - name: Start devservices early + uses: ./.github/actions/setup-devservices + with: + mode: default + - name: Setup sentry python env uses: ./.github/actions/setup-sentry id: setup with: mode: default + skip-devservices: 'true' + + - name: Wait for devservices + run: | + sentry init + ./.github/actions/setup-devservices/wait.sh - name: Run API docs tests run: | @@ -354,11 +366,22 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Start devservices early + uses: ./.github/actions/setup-devservices + with: + mode: default + - name: Setup sentry env uses: ./.github/actions/setup-sentry id: setup with: mode: default + skip-devservices: 'true' + + - name: Wait for devservices + run: | + sentry init + ./.github/actions/setup-devservices/wait.sh - name: run tests run: | @@ -392,11 +415,22 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Start devservices early + uses: ./.github/actions/setup-devservices + with: + mode: migrations + - name: Setup sentry env uses: ./.github/actions/setup-sentry id: setup with: mode: migrations + skip-devservices: 'true' + + - name: Wait for devservices + run: | + sentry init + ./.github/actions/setup-devservices/wait.sh - name: Run test env: @@ -468,10 +502,21 @@ jobs: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Start devservices early + uses: ./.github/actions/setup-devservices + with: + mode: backend-ci + - name: Setup sentry env uses: ./.github/actions/setup-sentry with: mode: backend-ci + skip-devservices: 'true' + + - name: Wait for devservices + run: | + sentry init + ./.github/actions/setup-devservices/wait.sh - name: Sync API Urls to TypeScript run: | @@ -494,11 +539,22 @@ jobs: - name: Checkout sentry uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Start devservices early + uses: ./.github/actions/setup-devservices + with: + mode: migrations + - name: Setup sentry env uses: ./.github/actions/setup-sentry id: setup with: mode: migrations + skip-devservices: 'true' + + - name: Wait for devservices + run: | + sentry init + ./.github/actions/setup-devservices/wait.sh - name: Migration & lockfile checks env: @@ -528,11 +584,22 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Start devservices early + uses: ./.github/actions/setup-devservices + with: + mode: migrations + - name: Setup sentry env uses: ./.github/actions/setup-sentry id: setup with: mode: migrations + skip-devservices: 'true' + + - name: Wait for devservices + run: | + sentry init + ./.github/actions/setup-devservices/wait.sh - name: Run test run: | diff --git a/.github/workflows/migrations-drift.yml b/.github/workflows/migrations-drift.yml index 37232c94b4fc0c..ca7b72ef3d370c 100644 --- a/.github/workflows/migrations-drift.yml +++ b/.github/workflows/migrations-drift.yml @@ -27,10 +27,21 @@ jobs: with: ref: master + - name: Start devservices early + uses: ./.github/actions/setup-devservices + with: + mode: migrations + - name: Setup sentry env uses: ./.github/actions/setup-sentry with: mode: migrations + skip-devservices: 'true' + + - name: Wait for devservices + run: | + sentry init + ./.github/actions/setup-devservices/wait.sh - name: Apply migrations env: diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index 359057e28a2fde..aecae374063213 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -47,10 +47,21 @@ jobs: with: ref: master + - name: Start devservices early + uses: ./.github/actions/setup-devservices + with: + mode: migrations + - name: Setup sentry env uses: ./.github/actions/setup-sentry with: mode: migrations + skip-devservices: 'true' + + - name: Wait for devservices + run: | + sentry init + ./.github/actions/setup-devservices/wait.sh - name: Apply migrations run: | diff --git a/.github/workflows/openapi-diff.yml b/.github/workflows/openapi-diff.yml index 9c510beafba3d9..556d036b53b456 100644 --- a/.github/workflows/openapi-diff.yml +++ b/.github/workflows/openapi-diff.yml @@ -27,12 +27,25 @@ jobs: token: ${{ github.token }} filters: .github/file-filters.yml + - name: Start devservices early + uses: ./.github/actions/setup-devservices + if: steps.changes.outputs.api_docs == 'true' + with: + mode: migrations + - name: Setup sentry env uses: ./.github/actions/setup-sentry with: mode: migrations + skip-devservices: 'true' if: steps.changes.outputs.api_docs == 'true' + - name: Wait for devservices + if: steps.changes.outputs.api_docs == 'true' + run: | + sentry init + ./.github/actions/setup-devservices/wait.sh + - name: Checkout getsentry/sentry-api-schema uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 if: steps.changes.outputs.api_docs == 'true' diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml index 35a7bad47fdef4..1ed5a0720e9af9 100644 --- a/.github/workflows/openapi.yml +++ b/.github/workflows/openapi.yml @@ -32,12 +32,25 @@ jobs: token: ${{ github.token }} filters: .github/file-filters.yml + - name: Start devservices early + uses: ./.github/actions/setup-devservices + if: steps.changes.outputs.api_docs == 'true' + with: + mode: migrations + - name: Setup sentry env uses: ./.github/actions/setup-sentry with: mode: migrations + skip-devservices: 'true' if: steps.changes.outputs.api_docs == 'true' + - name: Wait for devservices + if: steps.changes.outputs.api_docs == 'true' + run: | + sentry init + ./.github/actions/setup-devservices/wait.sh + - name: Checkout getsentry/sentry-api-schema if: steps.changes.outputs.api_docs == 'true' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 From 74ab45471ac90550a2f01791a413019607d7b53a Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 10:51:34 -0700 Subject: [PATCH 04/13] fix --- .github/workflows/acceptance.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index dbe9d9edf2f0e8..f088858e76458a 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -68,11 +68,6 @@ jobs: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 name: Checkout sentry - - name: Start devservices early - uses: ./.github/actions/setup-devservices - with: - mode: acceptance-ci - - name: Step configurations id: config run: | @@ -98,6 +93,11 @@ jobs: run: | make build-chartcuterie-config + - name: Start devservices early + uses: ./.github/actions/setup-devservices + with: + mode: acceptance-ci + - name: Setup sentry env uses: ./.github/actions/setup-sentry id: setup From 2dd7a0182031d99058aed7e7e825f63b018e8583 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 10:56:17 -0700 Subject: [PATCH 05/13] fix(ci): address review feedback for early devservices - Use strict shell (set -euo pipefail) at action level - Fix set -e preventing exit code file from being written in subshell - Add 10-minute timeout to Snuba bootstrap wait loop - Move devservices start after chartcuterie build in acceptance - Revert migrations/migrations-drift (checkout master first, action doesn't exist there yet) --- .github/actions/setup-devservices/action.yml | 4 ++-- .github/workflows/backend.yml | 9 ++++++++- .github/workflows/migrations-drift.yml | 11 ----------- .github/workflows/migrations.yml | 11 ----------- 4 files changed, 10 insertions(+), 25 deletions(-) diff --git a/.github/actions/setup-devservices/action.yml b/.github/actions/setup-devservices/action.yml index 496a7da5cdb717..061ff20d9b72e4 100644 --- a/.github/actions/setup-devservices/action.yml +++ b/.github/actions/setup-devservices/action.yml @@ -18,7 +18,7 @@ runs: enable-cache: false - name: Start devservices in background - shell: bash + shell: bash --noprofile --norc -euo pipefail {0} run: | DS_VERSION=$(python3 -c " import tomllib @@ -34,5 +34,5 @@ runs: uv pip install --python /tmp/ds-venv/bin/python -q \ --index-url https://pypi.devinfra.sentry.io/simple \ "devservices==${DS_VERSION}" - (timeout ${{ inputs.timeout-minutes }}m /tmp/ds-venv/bin/devservices up --mode ${{ inputs.mode }}; echo $? > /tmp/ds-exit) \ + (set +e; timeout ${{ inputs.timeout-minutes }}m /tmp/ds-venv/bin/devservices up --mode ${{ inputs.mode }}; echo $? > /tmp/ds-exit) \ > /tmp/ds.log 2>&1 & diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 330baf5fe5091d..912c6e75310f84 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -279,7 +279,14 @@ jobs: - name: Wait for Snuba bootstrap if: env.XDIST_PER_WORKER_SNUBA == '1' run: | - while [ ! -f /tmp/snuba-bootstrap-exit ]; do sleep 2; done + SECONDS=0 + while [ ! -f /tmp/snuba-bootstrap-exit ]; do + if [ $SECONDS -gt 600 ]; then + echo "::error::Timed out waiting for Snuba bootstrap after 600s" + exit 1 + fi + sleep 2 + done SNUBA_RC=$(cat /tmp/snuba-bootstrap-exit) if [ "$SNUBA_RC" -ne 0 ]; then echo "::error::Snuba per-worker bootstrap failed" diff --git a/.github/workflows/migrations-drift.yml b/.github/workflows/migrations-drift.yml index ca7b72ef3d370c..37232c94b4fc0c 100644 --- a/.github/workflows/migrations-drift.yml +++ b/.github/workflows/migrations-drift.yml @@ -27,21 +27,10 @@ jobs: with: ref: master - - name: Start devservices early - uses: ./.github/actions/setup-devservices - with: - mode: migrations - - name: Setup sentry env uses: ./.github/actions/setup-sentry with: mode: migrations - skip-devservices: 'true' - - - name: Wait for devservices - run: | - sentry init - ./.github/actions/setup-devservices/wait.sh - name: Apply migrations env: diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index aecae374063213..359057e28a2fde 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -47,21 +47,10 @@ jobs: with: ref: master - - name: Start devservices early - uses: ./.github/actions/setup-devservices - with: - mode: migrations - - name: Setup sentry env uses: ./.github/actions/setup-sentry with: mode: migrations - skip-devservices: 'true' - - - name: Wait for devservices - run: | - sentry init - ./.github/actions/setup-devservices/wait.sh - name: Apply migrations run: | From f9069db9f44758cbda4237e0c8028edf86aaeeb1 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 10:57:08 -0700 Subject: [PATCH 06/13] fix(ci): check devservices exit code before snuba Phase 2 --- .github/actions/setup-devservices/bootstrap-snuba.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/setup-devservices/bootstrap-snuba.sh b/.github/actions/setup-devservices/bootstrap-snuba.sh index 82e771505772c3..80e72859d42a99 100755 --- a/.github/actions/setup-devservices/bootstrap-snuba.sh +++ b/.github/actions/setup-devservices/bootstrap-snuba.sh @@ -68,6 +68,12 @@ echo "Phase 1 done (${SECONDS}s)" # Phase 2: Wait for devservices to finish, then swap snuba-snuba-1 for per-worker containers. while [ ! -f /tmp/ds-exit ]; do sleep 1; done +DS_RC=$(cat /tmp/ds-exit) +if [ "$DS_RC" -ne 0 ]; then + echo "::error::devservices failed (exit $DS_RC), skipping Phase 2" + echo 1 > /tmp/snuba-bootstrap-exit + exit 1 +fi docker stop snuba-snuba-1 || true From 01d6123efcf70afa39de480dc296944f73af5702 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 10:59:46 -0700 Subject: [PATCH 07/13] refinements --- .github/actions/setup-devservices/bootstrap-snuba.sh | 2 +- .github/actions/setup-devservices/wait.sh | 2 +- .github/workflows/acceptance.yml | 11 ++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/actions/setup-devservices/bootstrap-snuba.sh b/.github/actions/setup-devservices/bootstrap-snuba.sh index 80e72859d42a99..03888ba4b72bf8 100755 --- a/.github/actions/setup-devservices/bootstrap-snuba.sh +++ b/.github/actions/setup-devservices/bootstrap-snuba.sh @@ -68,7 +68,7 @@ echo "Phase 1 done (${SECONDS}s)" # Phase 2: Wait for devservices to finish, then swap snuba-snuba-1 for per-worker containers. while [ ! -f /tmp/ds-exit ]; do sleep 1; done -DS_RC=$(cat /tmp/ds-exit) +DS_RC=$(< /tmp/ds-exit) if [ "$DS_RC" -ne 0 ]; then echo "::error::devservices failed (exit $DS_RC), skipping Phase 2" echo 1 > /tmp/snuba-bootstrap-exit diff --git a/.github/actions/setup-devservices/wait.sh b/.github/actions/setup-devservices/wait.sh index 6f07708be84407..00c319469d7347 100755 --- a/.github/actions/setup-devservices/wait.sh +++ b/.github/actions/setup-devservices/wait.sh @@ -15,7 +15,7 @@ while [ ! -f /tmp/ds-exit ]; do sleep 2 done -DS_RC=$(cat /tmp/ds-exit) +DS_RC=$(< /tmp/ds-exit) if [ "$DS_RC" -ne 0 ]; then echo "::error::devservices up failed (exit $DS_RC)" cat /tmp/ds.log diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index f088858e76458a..652d2ca72b9392 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -68,6 +68,12 @@ jobs: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 name: Checkout sentry + - run: mkdir -p config/chartcuterie + + - uses: ./.github/actions/setup-devservices + with: + mode: acceptance-ci + - name: Step configurations id: config run: | @@ -93,11 +99,6 @@ jobs: run: | make build-chartcuterie-config - - name: Start devservices early - uses: ./.github/actions/setup-devservices - with: - mode: acceptance-ci - - name: Setup sentry env uses: ./.github/actions/setup-sentry id: setup From c8766fe0a4146ab2b65f70608bd71144b1bacde7 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 11:09:07 -0700 Subject: [PATCH 08/13] test change for backend --- .../test_organization_ai_conversations.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/sentry/api/endpoints/test_organization_ai_conversations.py b/tests/sentry/api/endpoints/test_organization_ai_conversations.py index 5b5c919236831b..2ea6354c062c3d 100644 --- a/tests/sentry/api/endpoints/test_organization_ai_conversations.py +++ b/tests/sentry/api/endpoints/test_organization_ai_conversations.py @@ -637,7 +637,7 @@ def test_complete_conversation_data_across_time_range(self) -> None: def test_first_input_last_output(self) -> None: """Test firstInput and lastOutput are correctly populated from ai_client spans""" - now = before_now(days=90).replace(microsecond=0) + now = before_now(days=11).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -699,7 +699,7 @@ def test_first_input_last_output(self) -> None: def test_no_ai_client_spans_filtered_out(self) -> None: """Test conversations without input/output are filtered out""" - now = before_now(days=91).replace(microsecond=0) + now = before_now(days=12).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -770,7 +770,7 @@ def test_query_filter(self) -> None: def test_conversation_with_user_data(self) -> None: """Test that user data is extracted from spans and returned in the response""" - now = before_now(days=100).replace(microsecond=0) + now = before_now(days=13).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -822,7 +822,7 @@ def test_conversation_with_user_data(self) -> None: def test_conversation_with_partial_user_data(self) -> None: """Test that user is returned even with partial user data""" - now = before_now(days=101).replace(microsecond=0) + now = before_now(days=14).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -857,7 +857,7 @@ def test_conversation_with_partial_user_data(self) -> None: def test_new_format_input_output_messages(self) -> None: """Test that new format gen_ai.input.messages and gen_ai.output.messages are parsed correctly""" - now = before_now(days=102).replace(microsecond=0) + now = before_now(days=16).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -904,7 +904,7 @@ def test_new_format_input_output_messages(self) -> None: def test_new_format_with_multiple_text_parts(self) -> None: """Test that multiple text parts are concatenated correctly""" - now = before_now(days=103).replace(microsecond=0) + now = before_now(days=17).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -955,7 +955,7 @@ def test_new_format_with_multiple_text_parts(self) -> None: def test_new_format_priority_over_old_format(self) -> None: """Test that new format attributes take priority over old format when both exist""" - now = before_now(days=104).replace(microsecond=0) + now = before_now(days=18).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1010,7 +1010,7 @@ def test_new_format_priority_over_old_format(self) -> None: def test_new_format_parts_structure(self) -> None: """Test that new format with parts structure works correctly""" - now = before_now(days=105).replace(microsecond=0) + now = before_now(days=19).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1055,7 +1055,7 @@ def test_new_format_parts_structure(self) -> None: def test_tool_names_populated(self) -> None: """Test that toolNames is populated with distinct tool names from tool spans""" - now = before_now(days=106).replace(microsecond=0) + now = before_now(days=21).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1128,7 +1128,7 @@ def test_tool_names_populated(self) -> None: def test_tool_errors_counted(self) -> None: """Test that toolErrors counts only failed tool spans""" - now = before_now(days=107).replace(microsecond=0) + now = before_now(days=22).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1212,7 +1212,7 @@ def test_tool_errors_counted(self) -> None: def test_empty_tool_names_when_no_tool_calls(self) -> None: """Test that toolNames is empty when there are no tool calls""" - now = before_now(days=108).replace(microsecond=0) + now = before_now(days=23).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1257,7 +1257,7 @@ def test_tokens_only_counted_from_ai_client_spans(self) -> None: This prevents double counting when both agent spans (invoke_agent) and their child ai_client spans have token/cost data. """ - now = before_now(days=109).replace(microsecond=0) + now = before_now(days=24).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex From 43254028bbd102ff398058aba6ccc168394729c7 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 11:15:33 -0700 Subject: [PATCH 09/13] Revert "test change for backend" This reverts commit c8766fe0a4146ab2b65f70608bd71144b1bacde7. --- .../test_organization_ai_conversations.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/sentry/api/endpoints/test_organization_ai_conversations.py b/tests/sentry/api/endpoints/test_organization_ai_conversations.py index 2ea6354c062c3d..5b5c919236831b 100644 --- a/tests/sentry/api/endpoints/test_organization_ai_conversations.py +++ b/tests/sentry/api/endpoints/test_organization_ai_conversations.py @@ -637,7 +637,7 @@ def test_complete_conversation_data_across_time_range(self) -> None: def test_first_input_last_output(self) -> None: """Test firstInput and lastOutput are correctly populated from ai_client spans""" - now = before_now(days=11).replace(microsecond=0) + now = before_now(days=90).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -699,7 +699,7 @@ def test_first_input_last_output(self) -> None: def test_no_ai_client_spans_filtered_out(self) -> None: """Test conversations without input/output are filtered out""" - now = before_now(days=12).replace(microsecond=0) + now = before_now(days=91).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -770,7 +770,7 @@ def test_query_filter(self) -> None: def test_conversation_with_user_data(self) -> None: """Test that user data is extracted from spans and returned in the response""" - now = before_now(days=13).replace(microsecond=0) + now = before_now(days=100).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -822,7 +822,7 @@ def test_conversation_with_user_data(self) -> None: def test_conversation_with_partial_user_data(self) -> None: """Test that user is returned even with partial user data""" - now = before_now(days=14).replace(microsecond=0) + now = before_now(days=101).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -857,7 +857,7 @@ def test_conversation_with_partial_user_data(self) -> None: def test_new_format_input_output_messages(self) -> None: """Test that new format gen_ai.input.messages and gen_ai.output.messages are parsed correctly""" - now = before_now(days=16).replace(microsecond=0) + now = before_now(days=102).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -904,7 +904,7 @@ def test_new_format_input_output_messages(self) -> None: def test_new_format_with_multiple_text_parts(self) -> None: """Test that multiple text parts are concatenated correctly""" - now = before_now(days=17).replace(microsecond=0) + now = before_now(days=103).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -955,7 +955,7 @@ def test_new_format_with_multiple_text_parts(self) -> None: def test_new_format_priority_over_old_format(self) -> None: """Test that new format attributes take priority over old format when both exist""" - now = before_now(days=18).replace(microsecond=0) + now = before_now(days=104).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1010,7 +1010,7 @@ def test_new_format_priority_over_old_format(self) -> None: def test_new_format_parts_structure(self) -> None: """Test that new format with parts structure works correctly""" - now = before_now(days=19).replace(microsecond=0) + now = before_now(days=105).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1055,7 +1055,7 @@ def test_new_format_parts_structure(self) -> None: def test_tool_names_populated(self) -> None: """Test that toolNames is populated with distinct tool names from tool spans""" - now = before_now(days=21).replace(microsecond=0) + now = before_now(days=106).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1128,7 +1128,7 @@ def test_tool_names_populated(self) -> None: def test_tool_errors_counted(self) -> None: """Test that toolErrors counts only failed tool spans""" - now = before_now(days=22).replace(microsecond=0) + now = before_now(days=107).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1212,7 +1212,7 @@ def test_tool_errors_counted(self) -> None: def test_empty_tool_names_when_no_tool_calls(self) -> None: """Test that toolNames is empty when there are no tool calls""" - now = before_now(days=23).replace(microsecond=0) + now = before_now(days=108).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1257,7 +1257,7 @@ def test_tokens_only_counted_from_ai_client_spans(self) -> None: This prevents double counting when both agent spans (invoke_agent) and their child ai_client spans have token/cost data. """ - now = before_now(days=24).replace(microsecond=0) + now = before_now(days=109).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex From 25e1ae1fd51ebb2086c0d77b5c18ca13ad9b804b Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 14:14:01 -0700 Subject: [PATCH 10/13] port to more resilient bootstrap-snuba.py --- .../setup-devservices/bootstrap-snuba.py | 246 ++++++++++++++++++ .../setup-devservices/bootstrap-snuba.sh | 114 -------- .github/workflows/backend.yml | 2 +- 3 files changed, 247 insertions(+), 115 deletions(-) create mode 100755 .github/actions/setup-devservices/bootstrap-snuba.py delete mode 100755 .github/actions/setup-devservices/bootstrap-snuba.sh diff --git a/.github/actions/setup-devservices/bootstrap-snuba.py b/.github/actions/setup-devservices/bootstrap-snuba.py new file mode 100755 index 00000000000000..acce05d1cc27bc --- /dev/null +++ b/.github/actions/setup-devservices/bootstrap-snuba.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Bootstrap per-worker Snuba instances for CI. + +Overlaps the expensive ClickHouse table setup with the devservices +health-check wait. + +Phase 1 (early): As soon as ClickHouse is accepting queries, create + per-worker databases and run ``snuba bootstrap --force``. +Phase 2 (after devservices): Stop snuba-snuba-1 and start per-worker + API containers. We must wait for devservices to finish first — + stopping the container while devservices is health-checking it would + cause a timeout. + +Requires: XDIST_WORKERS env var +Reads: /tmp/ds-exit (written by setup-devservices/wait.sh) +Writes: /tmp/snuba-bootstrap-exit +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from functools import partial +from pathlib import Path +from typing import Any, Callable +from urllib.error import URLError +from urllib.request import urlopen + +DS_EXIT = Path("/tmp/ds-exit") +SNUBA_EXIT = Path("/tmp/snuba-bootstrap-exit") + +SNUBA_ENV = { + "CLICKHOUSE_HOST": "clickhouse", + "CLICKHOUSE_PORT": "9000", + "CLICKHOUSE_HTTP_PORT": "8123", + "DEFAULT_BROKERS": "kafka:9093", + "REDIS_HOST": "redis", + "REDIS_PORT": "6379", + "REDIS_DB": "1", + "SNUBA_SETTINGS": "docker", +} + +ENV_ARGS = [flag for k, v in SNUBA_ENV.items() for flag in ("-e", f"{k}={v}")] + + +def retry( + fn: Callable[[], Any], *, attempts: int = 3, delay: int = 5, label: str = "operation" +) -> Any: + for i in range(attempts): + try: + return fn() + except Exception: + if i == attempts - 1: + raise + log(f"{label} failed (attempt {i + 1}/{attempts}), retrying in {delay}s...") + time.sleep(delay) + + +def log(msg: str) -> None: + print(msg, flush=True) + + +def fail(msg: str) -> None: + log(f"::error::{msg}") + SNUBA_EXIT.write_text("1") + sys.exit(1) + + +def http_ok(url: str) -> bool: + try: + with urlopen(url, timeout=3): + return True + except (URLError, OSError): + return False + + +def docker( + *args: str, check: bool = False, timeout: int | None = None +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["docker", *args], capture_output=True, text=True, check=check, timeout=timeout + ) + + +def docker_inspect(container: str, fmt: str) -> str: + r = docker("inspect", container, "--format", fmt) + return r.stdout.strip() if r.returncode == 0 else "" + + +def inspect_snuba_container() -> tuple[str, str]: + image = docker_inspect("snuba-snuba-1", "{{.Config.Image}}") + network = docker_inspect( + "snuba-snuba-1", + "{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}", + ) + if not image or not network: + fail("Could not inspect snuba-snuba-1 container") + return image, network + + +def run_parallel(fn: Callable[[int], Any], workers: range, *, fail_fast: bool = True) -> int: + """Run fn(i) in parallel for each i in workers. Returns 0 on full success.""" + rc = 0 + with ThreadPoolExecutor(max_workers=len(workers)) as pool: + futs = {pool.submit(fn, i): i for i in workers} + for fut in as_completed(futs): + try: + fut.result() + except Exception as e: + if fail_fast: + fail(str(e)) + log(f"ERROR: {e}") + rc = 1 + return rc + + +def wait_for_prerequisites(timeout: int = 300) -> None: + log("Waiting for ClickHouse and Snuba container...") + start = time.monotonic() + while True: + if time.monotonic() - start > timeout: + fail("Timed out waiting for Snuba bootstrap prerequisites") + if http_ok("http://localhost:8123/") and docker_inspect("snuba-snuba-1", "{{.Id}}"): + break + time.sleep(2) + log(f"Prerequisites ready ({time.monotonic() - start:.0f}s)") + + +def wait_for_devservices(timeout: int = 300) -> None: + start = time.monotonic() + while not DS_EXIT.exists(): + if time.monotonic() - start > timeout: + fail("Timed out waiting for devservices to finish") + time.sleep(1) + rc = int(DS_EXIT.read_text().strip()) + if rc != 0: + fail(f"devservices failed (exit {rc}), skipping Phase 2") + + +def bootstrap_worker(worker_id: int, *, image: str, network: str) -> None: + """Create a ClickHouse database and run snuba bootstrap.""" + db = f"default_gw{worker_id}" + + def create_db() -> None: + with urlopen( + "http://localhost:8123/", f"CREATE DATABASE IF NOT EXISTS {db}".encode(), timeout=30 + ): + pass + + retry(create_db, label=f"CREATE DATABASE {db}") + + def run_bootstrap() -> None: + r = docker( + "run", + "--rm", + "--network", + network, + "-e", + f"CLICKHOUSE_DATABASE={db}", + *ENV_ARGS, + image, + "bootstrap", + "--force", + ) + for line in (r.stdout + r.stderr).strip().splitlines()[-3:]: + log(line) + if r.returncode != 0: + raise RuntimeError(f"snuba bootstrap failed for worker {worker_id}") + + retry(run_bootstrap, label=f"snuba bootstrap gw{worker_id}") + + +def start_worker_container(worker_id: int, *, image: str, network: str) -> None: + """Start a per-worker Snuba API container and wait for health.""" + db = f"default_gw{worker_id}" + port = 1230 + worker_id + name = f"snuba-gw{worker_id}" + + docker("rm", "-f", name) + + r = docker( + "run", + "-d", + "--name", + name, + "--network", + network, + "-p", + f"{port}:1218", + "-e", + f"CLICKHOUSE_DATABASE={db}", + *ENV_ARGS, + "-e", + "DEBUG=1", + image, + "api", + ) + if r.returncode != 0: + raise RuntimeError(f"docker run {name} failed: {r.stderr.strip()}") + + for attempt in range(1, 31): + if http_ok(f"http://127.0.0.1:{port}/health"): + log(f"{name} healthy on port {port}") + return + if attempt == 30: + r = docker("logs", name) + for line in (r.stdout + r.stderr).strip().splitlines()[-20:]: + log(line) + raise RuntimeError(f"{name} failed health check after 30 attempts") + time.sleep(2) + + +def main() -> None: + workers_str = os.environ.get("XDIST_WORKERS") + if not workers_str: + fail("XDIST_WORKERS must be set") + workers = range(int(workers_str)) + start = time.monotonic() + + wait_for_prerequisites() + image, network = inspect_snuba_container() + + log("Phase 1: bootstrapping ClickHouse databases") + run_parallel(partial(bootstrap_worker, image=image, network=network), workers) + log(f"Phase 1 done ({time.monotonic() - start:.0f}s)") + + wait_for_devservices() + docker("stop", "snuba-snuba-1", timeout=30) + + log("Phase 2: starting per-worker Snuba API containers") + rc = run_parallel( + partial(start_worker_container, image=image, network=network), + workers, + fail_fast=False, + ) + + log(f"Snuba bootstrap complete ({time.monotonic() - start:.0f}s total)") + SNUBA_EXIT.write_text(str(rc)) + sys.exit(rc) + + +if __name__ == "__main__": + main() diff --git a/.github/actions/setup-devservices/bootstrap-snuba.sh b/.github/actions/setup-devservices/bootstrap-snuba.sh deleted file mode 100755 index 03888ba4b72bf8..00000000000000 --- a/.github/actions/setup-devservices/bootstrap-snuba.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Bootstrap per-worker Snuba instances, overlapping the expensive ClickHouse -# table setup with the devservices health-check wait. -# -# Phase 1 (early): As soon as ClickHouse is accepting queries, create per-worker -# databases and run `snuba bootstrap --force`. This is the slow part. -# Phase 2 (after devservices): Stop snuba-snuba-1 and start per-worker API -# containers. We must wait for devservices to finish first — stopping the -# container while devservices is health-checking it would cause a timeout. -# -# Requires: XDIST_WORKERS env var -# Reads: /tmp/ds-exit (written by setup-devservices/wait.sh) -# Writes: /tmp/snuba-bootstrap-exit - -WORKERS=${XDIST_WORKERS:?XDIST_WORKERS must be set} - -echo "Waiting for ClickHouse and Snuba container..." -SECONDS=0 -while true; do - if [ $SECONDS -gt 300 ]; then - echo "::error::Timed out waiting for Snuba bootstrap prerequisites" - echo 1 > /tmp/snuba-bootstrap-exit - exit 1 - fi - if curl -sf 'http://localhost:8123/' > /dev/null 2>&1 \ - && docker inspect snuba-snuba-1 > /dev/null 2>&1; then - break - fi - sleep 2 -done -echo "Prerequisites ready (${SECONDS}s)" - -SNUBA_IMAGE=$(docker inspect snuba-snuba-1 --format '{{.Config.Image}}') -SNUBA_NETWORK=$(docker inspect snuba-snuba-1 --format '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}') -if [ -z "$SNUBA_IMAGE" ] || [ -z "$SNUBA_NETWORK" ]; then - echo "::error::Could not inspect snuba-snuba-1 container" - echo 1 > /tmp/snuba-bootstrap-exit - exit 1 -fi - -SNUBA_ENV=( - -e "CLICKHOUSE_HOST=clickhouse" -e "CLICKHOUSE_PORT=9000" -e "CLICKHOUSE_HTTP_PORT=8123" - -e "DEFAULT_BROKERS=kafka:9093" -e "REDIS_HOST=redis" -e "REDIS_PORT=6379" -e "REDIS_DB=1" - -e "SNUBA_SETTINGS=docker" -) - -# Phase 1: Create databases and run bootstrap (the expensive part). -# This can safely run while devservices is still health-checking containers. -echo "Phase 1: bootstrapping ClickHouse databases" -BOOTSTRAP_PIDS=() -for i in $(seq 0 $(( WORKERS - 1 ))); do - ( - WORKER_DB="default_gw${i}" - curl -sf 'http://localhost:8123/' --data-binary "CREATE DATABASE IF NOT EXISTS ${WORKER_DB}" - docker run --rm --network "$SNUBA_NETWORK" \ - -e "CLICKHOUSE_DATABASE=${WORKER_DB}" "${SNUBA_ENV[@]}" \ - "$SNUBA_IMAGE" bootstrap --force 2>&1 | tail -3 - ) & - BOOTSTRAP_PIDS+=($!) -done - -for pid in "${BOOTSTRAP_PIDS[@]}"; do - wait "$pid" || { echo "ERROR: Snuba bootstrap (PID $pid) failed"; echo 1 > /tmp/snuba-bootstrap-exit; exit 1; } -done -echo "Phase 1 done (${SECONDS}s)" - -# Phase 2: Wait for devservices to finish, then swap snuba-snuba-1 for per-worker containers. -while [ ! -f /tmp/ds-exit ]; do sleep 1; done -DS_RC=$(< /tmp/ds-exit) -if [ "$DS_RC" -ne 0 ]; then - echo "::error::devservices failed (exit $DS_RC), skipping Phase 2" - echo 1 > /tmp/snuba-bootstrap-exit - exit 1 -fi - -docker stop snuba-snuba-1 || true - -echo "Phase 2: starting per-worker Snuba API containers" -GW_PIDS=() -for i in $(seq 0 $(( WORKERS - 1 ))); do - ( - WORKER_DB="default_gw${i}" - WORKER_PORT=$((1230 + i)) - docker run -d --name "snuba-gw${i}" --network "$SNUBA_NETWORK" \ - -p "${WORKER_PORT}:1218" \ - -e "CLICKHOUSE_DATABASE=${WORKER_DB}" "${SNUBA_ENV[@]}" \ - -e "DEBUG=1" "$SNUBA_IMAGE" api - - for attempt in $(seq 1 30); do - if curl -sf "http://127.0.0.1:${WORKER_PORT}/health" > /dev/null 2>&1; then - echo "snuba-gw${i} healthy on port ${WORKER_PORT}" - break - fi - if [ "$attempt" -eq 30 ]; then - echo "ERROR: snuba-gw${i} failed health check after 30 attempts" - docker logs "snuba-gw${i}" 2>&1 | tail -20 || true - exit 1 - fi - sleep 2 - done - ) & - GW_PIDS+=($!) -done - -RC=0 -for pid in "${GW_PIDS[@]}"; do - wait "$pid" || { echo "ERROR: Snuba gateway (PID $pid) failed"; RC=1; } -done - -echo "Snuba bootstrap complete (${SECONDS}s total)" -echo $RC > /tmp/snuba-bootstrap-exit -exit $RC diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 912c6e75310f84..fbcde1793f949e 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -265,7 +265,7 @@ jobs: # ClickHouse readiness itself, so it can begin before all containers # are healthy, overlapping with the remaining devservices health checks. if [ "${XDIST_PER_WORKER_SNUBA}" = "1" ]; then - ./.github/actions/setup-devservices/bootstrap-snuba.sh & + python3 ./.github/actions/setup-devservices/bootstrap-snuba.py & fi ./.github/actions/setup-devservices/wait.sh From 6d20efbcd403cbd6125fc790ecef84b4200f343d Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 14:14:07 -0700 Subject: [PATCH 11/13] Reapply "test change for backend" This reverts commit 43254028bbd102ff398058aba6ccc168394729c7. --- .../test_organization_ai_conversations.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/sentry/api/endpoints/test_organization_ai_conversations.py b/tests/sentry/api/endpoints/test_organization_ai_conversations.py index 5b5c919236831b..2ea6354c062c3d 100644 --- a/tests/sentry/api/endpoints/test_organization_ai_conversations.py +++ b/tests/sentry/api/endpoints/test_organization_ai_conversations.py @@ -637,7 +637,7 @@ def test_complete_conversation_data_across_time_range(self) -> None: def test_first_input_last_output(self) -> None: """Test firstInput and lastOutput are correctly populated from ai_client spans""" - now = before_now(days=90).replace(microsecond=0) + now = before_now(days=11).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -699,7 +699,7 @@ def test_first_input_last_output(self) -> None: def test_no_ai_client_spans_filtered_out(self) -> None: """Test conversations without input/output are filtered out""" - now = before_now(days=91).replace(microsecond=0) + now = before_now(days=12).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -770,7 +770,7 @@ def test_query_filter(self) -> None: def test_conversation_with_user_data(self) -> None: """Test that user data is extracted from spans and returned in the response""" - now = before_now(days=100).replace(microsecond=0) + now = before_now(days=13).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -822,7 +822,7 @@ def test_conversation_with_user_data(self) -> None: def test_conversation_with_partial_user_data(self) -> None: """Test that user is returned even with partial user data""" - now = before_now(days=101).replace(microsecond=0) + now = before_now(days=14).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -857,7 +857,7 @@ def test_conversation_with_partial_user_data(self) -> None: def test_new_format_input_output_messages(self) -> None: """Test that new format gen_ai.input.messages and gen_ai.output.messages are parsed correctly""" - now = before_now(days=102).replace(microsecond=0) + now = before_now(days=16).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -904,7 +904,7 @@ def test_new_format_input_output_messages(self) -> None: def test_new_format_with_multiple_text_parts(self) -> None: """Test that multiple text parts are concatenated correctly""" - now = before_now(days=103).replace(microsecond=0) + now = before_now(days=17).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -955,7 +955,7 @@ def test_new_format_with_multiple_text_parts(self) -> None: def test_new_format_priority_over_old_format(self) -> None: """Test that new format attributes take priority over old format when both exist""" - now = before_now(days=104).replace(microsecond=0) + now = before_now(days=18).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1010,7 +1010,7 @@ def test_new_format_priority_over_old_format(self) -> None: def test_new_format_parts_structure(self) -> None: """Test that new format with parts structure works correctly""" - now = before_now(days=105).replace(microsecond=0) + now = before_now(days=19).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1055,7 +1055,7 @@ def test_new_format_parts_structure(self) -> None: def test_tool_names_populated(self) -> None: """Test that toolNames is populated with distinct tool names from tool spans""" - now = before_now(days=106).replace(microsecond=0) + now = before_now(days=21).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1128,7 +1128,7 @@ def test_tool_names_populated(self) -> None: def test_tool_errors_counted(self) -> None: """Test that toolErrors counts only failed tool spans""" - now = before_now(days=107).replace(microsecond=0) + now = before_now(days=22).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1212,7 +1212,7 @@ def test_tool_errors_counted(self) -> None: def test_empty_tool_names_when_no_tool_calls(self) -> None: """Test that toolNames is empty when there are no tool calls""" - now = before_now(days=108).replace(microsecond=0) + now = before_now(days=23).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1257,7 +1257,7 @@ def test_tokens_only_counted_from_ai_client_spans(self) -> None: This prevents double counting when both agent spans (invoke_agent) and their child ai_client spans have token/cost data. """ - now = before_now(days=109).replace(microsecond=0) + now = before_now(days=24).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex From 569c9a77d9551badc58e05e1ef0bccf9e472b9da Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 14:18:28 -0700 Subject: [PATCH 12/13] Revert "Reapply "test change for backend"" This reverts commit 6d20efbcd403cbd6125fc790ecef84b4200f343d. --- .../test_organization_ai_conversations.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/sentry/api/endpoints/test_organization_ai_conversations.py b/tests/sentry/api/endpoints/test_organization_ai_conversations.py index 2ea6354c062c3d..5b5c919236831b 100644 --- a/tests/sentry/api/endpoints/test_organization_ai_conversations.py +++ b/tests/sentry/api/endpoints/test_organization_ai_conversations.py @@ -637,7 +637,7 @@ def test_complete_conversation_data_across_time_range(self) -> None: def test_first_input_last_output(self) -> None: """Test firstInput and lastOutput are correctly populated from ai_client spans""" - now = before_now(days=11).replace(microsecond=0) + now = before_now(days=90).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -699,7 +699,7 @@ def test_first_input_last_output(self) -> None: def test_no_ai_client_spans_filtered_out(self) -> None: """Test conversations without input/output are filtered out""" - now = before_now(days=12).replace(microsecond=0) + now = before_now(days=91).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -770,7 +770,7 @@ def test_query_filter(self) -> None: def test_conversation_with_user_data(self) -> None: """Test that user data is extracted from spans and returned in the response""" - now = before_now(days=13).replace(microsecond=0) + now = before_now(days=100).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -822,7 +822,7 @@ def test_conversation_with_user_data(self) -> None: def test_conversation_with_partial_user_data(self) -> None: """Test that user is returned even with partial user data""" - now = before_now(days=14).replace(microsecond=0) + now = before_now(days=101).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -857,7 +857,7 @@ def test_conversation_with_partial_user_data(self) -> None: def test_new_format_input_output_messages(self) -> None: """Test that new format gen_ai.input.messages and gen_ai.output.messages are parsed correctly""" - now = before_now(days=16).replace(microsecond=0) + now = before_now(days=102).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -904,7 +904,7 @@ def test_new_format_input_output_messages(self) -> None: def test_new_format_with_multiple_text_parts(self) -> None: """Test that multiple text parts are concatenated correctly""" - now = before_now(days=17).replace(microsecond=0) + now = before_now(days=103).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -955,7 +955,7 @@ def test_new_format_with_multiple_text_parts(self) -> None: def test_new_format_priority_over_old_format(self) -> None: """Test that new format attributes take priority over old format when both exist""" - now = before_now(days=18).replace(microsecond=0) + now = before_now(days=104).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1010,7 +1010,7 @@ def test_new_format_priority_over_old_format(self) -> None: def test_new_format_parts_structure(self) -> None: """Test that new format with parts structure works correctly""" - now = before_now(days=19).replace(microsecond=0) + now = before_now(days=105).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1055,7 +1055,7 @@ def test_new_format_parts_structure(self) -> None: def test_tool_names_populated(self) -> None: """Test that toolNames is populated with distinct tool names from tool spans""" - now = before_now(days=21).replace(microsecond=0) + now = before_now(days=106).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1128,7 +1128,7 @@ def test_tool_names_populated(self) -> None: def test_tool_errors_counted(self) -> None: """Test that toolErrors counts only failed tool spans""" - now = before_now(days=22).replace(microsecond=0) + now = before_now(days=107).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1212,7 +1212,7 @@ def test_tool_errors_counted(self) -> None: def test_empty_tool_names_when_no_tool_calls(self) -> None: """Test that toolNames is empty when there are no tool calls""" - now = before_now(days=23).replace(microsecond=0) + now = before_now(days=108).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex @@ -1257,7 +1257,7 @@ def test_tokens_only_counted_from_ai_client_spans(self) -> None: This prevents double counting when both agent spans (invoke_agent) and their child ai_client spans have token/cost data. """ - now = before_now(days=24).replace(microsecond=0) + now = before_now(days=109).replace(microsecond=0) conversation_id = uuid4().hex trace_id = uuid4().hex From f1b04e1c242d62362fed111d9ee659b426c1bb97 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 7 Apr 2026 14:26:22 -0700 Subject: [PATCH 13/13] fix --- .github/actions/setup-devservices/bootstrap-snuba.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-devservices/bootstrap-snuba.py b/.github/actions/setup-devservices/bootstrap-snuba.py index acce05d1cc27bc..c0f4ea8a7e7bc2 100755 --- a/.github/actions/setup-devservices/bootstrap-snuba.py +++ b/.github/actions/setup-devservices/bootstrap-snuba.py @@ -228,7 +228,11 @@ def main() -> None: log(f"Phase 1 done ({time.monotonic() - start:.0f}s)") wait_for_devservices() - docker("stop", "snuba-snuba-1", timeout=30) + try: + docker("stop", "snuba-snuba-1", timeout=30) + except subprocess.TimeoutExpired: + log("WARNING: docker stop snuba-snuba-1 timed out, killing") + docker("kill", "snuba-snuba-1") log("Phase 2: starting per-worker Snuba API containers") rc = run_parallel(