fix: properly check that only the last message is kept when doing multi-turn conversations #187
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR Test Comparison | |
| on: | |
| pull_request: | |
| branches: [main] | |
| # Cancel in-progress runs for the same PR | |
| concurrency: | |
| group: pr-tests-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| jobs: | |
| test-main: | |
| name: Test (main) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout main branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| submodules: true | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| cache: "npm" | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install uv (Python package manager) | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Get latest Sentry SDK versions | |
| id: sentry-versions | |
| run: | | |
| echo "js=$(npm view @sentry/node version)" >> $GITHUB_OUTPUT | |
| echo "python=$(curl -s https://pypi.org/pypi/sentry-sdk/json | python3 -c 'import sys,json; print(json.load(sys.stdin)["info"]["version"])')" >> $GITHUB_OUTPUT | |
| echo "php=$(curl -s https://repo.packagist.org/p2/sentry/sentry-laravel.json | python3 -c 'import sys,json; print(json.load(sys.stdin)["packages"]["sentry/sentry-laravel"][0]["version"])')" >> $GITHUB_OUTPUT | |
| - name: Cache Node test environments | |
| uses: actions/cache@v4 | |
| with: | |
| path: runs/node | |
| key: node-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/node/**/config.json') }} | |
| restore-keys: | | |
| node-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}- | |
| - name: Cache Browser test environments | |
| uses: actions/cache@v4 | |
| with: | |
| path: runs/browser | |
| key: browser-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/browser/**/config.json') }} | |
| restore-keys: | | |
| browser-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}- | |
| - name: Cache Next.js test environments | |
| uses: actions/cache@v4 | |
| with: | |
| path: runs/nextjs | |
| key: nextjs-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/nextjs/**/config.json') }} | |
| restore-keys: | | |
| nextjs-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}- | |
| - name: Cache Python test environments | |
| uses: actions/cache@v4 | |
| with: | |
| path: runs/python | |
| key: python-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.python }}-${{ hashFiles('src/runner/templates/**/python/**/config.json') }} | |
| restore-keys: | | |
| python-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.python }}- | |
| - name: Cache PHP test environments | |
| uses: actions/cache@v4 | |
| with: | |
| path: runs/php | |
| key: php-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.php }}-${{ hashFiles('src/runner/templates/**/php/**/config.json') }} | |
| restore-keys: | | |
| php-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.php }}- | |
| - name: Setup PHP | |
| uses: shivammathur/setup-php@v2 | |
| with: | |
| php-version: "8.4" | |
| tools: composer | |
| - name: Install dependencies and build | |
| run: | | |
| npm ci | |
| npm run build | |
| - name: Install Playwright browsers (for browser platform only) | |
| run: npx playwright install --with-deps chromium | |
| - name: Run tests | |
| id: run-tests | |
| continue-on-error: true | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| GOOGLE_GENAI_API_KEY: ${{ secrets.GOOGLE_GENAI_API_KEY }} | |
| CO_API_KEY: ${{ secrets.CO_API_KEY }} | |
| run: npm run test -- -j=12 | |
| - name: Upload CTRF report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ctrf-main | |
| path: test-results/ctrf-report-*.json | |
| retention-days: 1 | |
| test-pr: | |
| name: Test (PR) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| cache: "npm" | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install uv (Python package manager) | |
| uses: astral-sh/setup-uv@v5 | |
| - name: Get latest Sentry SDK versions | |
| id: sentry-versions | |
| run: | | |
| echo "js=$(npm view @sentry/node version)" >> $GITHUB_OUTPUT | |
| echo "python=$(curl -s https://pypi.org/pypi/sentry-sdk/json | python3 -c 'import sys,json; print(json.load(sys.stdin)["info"]["version"])')" >> $GITHUB_OUTPUT | |
| echo "php=$(curl -s https://repo.packagist.org/p2/sentry/sentry-laravel.json | python3 -c 'import sys,json; print(json.load(sys.stdin)["packages"]["sentry/sentry-laravel"][0]["version"])')" >> $GITHUB_OUTPUT | |
| - name: Cache Node test environments | |
| uses: actions/cache@v4 | |
| with: | |
| path: runs/node | |
| key: node-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/node/**/config.json') }} | |
| restore-keys: | | |
| node-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}- | |
| - name: Cache Browser test environments | |
| uses: actions/cache@v4 | |
| with: | |
| path: runs/browser | |
| key: browser-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/browser/**/config.json') }} | |
| restore-keys: | | |
| browser-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}- | |
| - name: Cache Next.js test environments | |
| uses: actions/cache@v4 | |
| with: | |
| path: runs/nextjs | |
| key: nextjs-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/nextjs/**/config.json') }} | |
| restore-keys: | | |
| nextjs-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}- | |
| - name: Cache Python test environments | |
| uses: actions/cache@v4 | |
| with: | |
| path: runs/python | |
| key: python-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.python }}-${{ hashFiles('src/runner/templates/**/python/**/config.json') }} | |
| restore-keys: | | |
| python-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.python }}- | |
| - name: Cache PHP test environments | |
| uses: actions/cache@v4 | |
| with: | |
| path: runs/php | |
| key: php-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.php }}-${{ hashFiles('src/runner/templates/**/php/**/config.json') }} | |
| restore-keys: | | |
| php-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.php }}- | |
| - name: Setup PHP | |
| uses: shivammathur/setup-php@v2 | |
| with: | |
| php-version: "8.4" | |
| tools: composer | |
| - name: Install dependencies and build | |
| run: | | |
| npm ci | |
| npm run build | |
| - name: Install Playwright browsers (for browser platform only) | |
| run: npx playwright install --with-deps chromium | |
| - name: Run tests | |
| id: run-tests | |
| continue-on-error: true | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| GOOGLE_GENAI_API_KEY: ${{ secrets.GOOGLE_GENAI_API_KEY }} | |
| CO_API_KEY: ${{ secrets.CO_API_KEY }} | |
| run: npm run test -- -j=12 | |
| - name: Upload CTRF report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ctrf-pr | |
| path: test-results/ctrf-report-*.json | |
| retention-days: 1 | |
| compare-results: | |
| name: Compare Results | |
| needs: [test-main, test-pr] | |
| runs-on: ubuntu-latest | |
| if: always() && needs.test-main.result != 'cancelled' && needs.test-pr.result != 'cancelled' | |
| steps: | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v4 | |
| - name: Download main CTRF report | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ctrf-main | |
| path: ctrf-main | |
| - name: Download PR CTRF report | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ctrf-pr | |
| path: ctrf-pr | |
| - name: Run comparison | |
| id: compare | |
| run: | | |
| # Find the latest CTRF reports (they have timestamps in filename) | |
| MAIN_REPORT=$(ls -t ctrf-main/ctrf-report-*.json 2>/dev/null | head -1) | |
| PR_REPORT=$(ls -t ctrf-pr/ctrf-report-*.json 2>/dev/null | head -1) | |
| if [ -z "$MAIN_REPORT" ] || [ -z "$PR_REPORT" ]; then | |
| echo "Missing CTRF reports" | |
| echo "Main: $MAIN_REPORT" | |
| echo "PR: $PR_REPORT" | |
| exit 1 | |
| fi | |
| node .github/scripts/compare-ctrf.cjs \ | |
| "$MAIN_REPORT" \ | |
| "$PR_REPORT" \ | |
| comparison-output.md | |
| # Check if there are regressions | |
| if grep -q "^HAS_REGRESSIONS=true$" comparison-output.env 2>/dev/null; then | |
| echo "has_regressions=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_regressions=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Write job summary | |
| run: cat comparison-output.md >> $GITHUB_STEP_SUMMARY | |
| - name: Find existing comment | |
| uses: peter-evans/find-comment@v3 | |
| id: find-comment | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-author: "github-actions[bot]" | |
| body-includes: "AI SDK Integration Test Results" | |
| - name: Create or update PR comment | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-id: ${{ steps.find-comment.outputs.comment-id }} | |
| edit-mode: replace | |
| body-path: comparison-output.md | |
| - name: Fail if regressions | |
| if: steps.compare.outputs.has_regressions == 'true' | |
| run: | | |
| echo "::error::Test regressions detected! Tests that were passing on main are now failing." | |
| exit 1 |