Run the integration test suites from tests/integration in replay mode #8
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests (Replay) | |
| run-name: Run the integration test suites from tests/integration in replay mode | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| pull_request: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| types: [opened, synchronize, reopened] | |
| paths: | |
| - 'src/llama_stack/**' | |
| - '!src/llama_stack_ui/**' | |
| - 'tests/**' | |
| - 'uv.lock' | |
| - 'pyproject.toml' | |
| - '.github/workflows/integration-tests.yml' # This workflow | |
| - '.github/actions/setup-ollama/action.yml' | |
| - '.github/actions/setup-test-environment/action.yml' | |
| - '.github/actions/run-and-record-tests/action.yml' | |
| - 'scripts/integration-tests.sh' | |
| - 'scripts/generate_ci_matrix.py' | |
| schedule: | |
| # If changing the cron schedule, update the provider in the test-matrix job | |
| - cron: '0 0 * * *' # (test latest client) Daily at 12 AM UTC | |
| workflow_dispatch: | |
| inputs: | |
| test-all-client-versions: | |
| description: 'Test against both the latest and published versions' | |
| type: boolean | |
| default: false | |
| test-setup: | |
| description: 'Test against a specific setup' | |
| type: string | |
| default: 'ollama' | |
| workflow_call: | |
| inputs: | |
| sdk_install_url: | |
| required: false | |
| type: string | |
| description: 'URL to install Python SDK from (for testing preview builds)' | |
| matrix_key: | |
| required: false | |
| type: string | |
| default: 'default' | |
| description: 'Matrix configuration key from ci_matrix.json (e.g., "default", "stainless")' | |
| pr_head_sha: | |
| required: false | |
| type: string | |
| description: 'The SHA of the pull request head to checkout' | |
| pr_head_ref: | |
| required: false | |
| type: string | |
| description: 'The branch name of the pull request head (for recording commits)' | |
| is_fork_pr: | |
| required: false | |
| type: boolean | |
| default: false | |
| description: 'Whether this is a fork PR (cannot push recordings to forks)' | |
| test-all-client-versions: | |
| required: false | |
| type: boolean | |
| default: false | |
| description: 'Test against both the latest and published versions' | |
| concurrency: | |
| # Skip concurrency for pushes to main - each commit should be tested independently | |
| group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| generate-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || github.event.pull_request.head.sha || github.sha }} | |
| - name: Generate test matrix | |
| id: set-matrix | |
| run: | | |
| # Generate matrix from CI_MATRIX in tests/integration/ci_matrix.json | |
| # Supports schedule-based, manual input, and workflow_call overrides | |
| MATRIX=$(PYTHONPATH=. python3 scripts/generate_ci_matrix.py \ | |
| --schedule "${{ github.event.schedule }}" \ | |
| --test-setup "${{ github.event.inputs.test-setup || '' }}" \ | |
| --matrix-key "${{ inputs.matrix_key || 'default' }}") | |
| echo "matrix=$MATRIX" >> $GITHUB_OUTPUT | |
| echo "Generated matrix: $MATRIX" | |
| run-replay-mode-tests: | |
| needs: generate-matrix | |
| runs-on: ubuntu-latest | |
| name: ${{ format('Integration Tests ({0}, {1}, {2}, client={3}, {4})', matrix.client, matrix.config.setup, matrix.python-version, matrix.client-version, matrix.config.suite) }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| client: [library, docker, server] | |
| # Use Python 3.13 only on nightly schedule (daily latest client test), otherwise use 3.12 | |
| python-version: ${{ github.event.schedule == '0 0 * * *' && fromJSON('["3.12", "3.13"]') || fromJSON('["3.12"]') }} | |
| node-version: [22] | |
| client-version: ${{ (github.event.schedule == '0 0 * * *' || github.event.inputs.test-all-client-versions == 'true' || inputs.test-all-client-versions == true) && fromJSON('["published", "latest"]') || fromJSON('["latest"]') }} | |
| # Test configurations: Generated from CI_MATRIX in tests/integration/ci_matrix.json | |
| # See scripts/generate_ci_matrix.py for generation logic | |
| config: ${{ fromJSON(needs.generate-matrix.outputs.matrix).include }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| ref: ${{ inputs.pr_head_sha || github.event.pull_request.head.sha || github.sha }} | |
| - name: Setup test environment | |
| if: ${{ matrix.config.allowed_clients == null || contains(matrix.config.allowed_clients, matrix.client) }} | |
| uses: ./.github/actions/setup-test-environment | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| client-version: ${{ matrix.client-version }} | |
| sdk_install_url: ${{ inputs.sdk_install_url || '' }} | |
| setup: ${{ matrix.config.setup }} | |
| suite: ${{ matrix.config.suite }} | |
| inference-mode: ${{ matrix.config.inference_mode || 'replay' }} | |
| - name: Setup Node.js for TypeScript client tests | |
| if: ${{ matrix.client == 'server' }} | |
| uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 | |
| with: | |
| node-version: ${{matrix.node-version}} | |
| cache: 'npm' | |
| cache-dependency-path: tests/integration/client-typescript/package-lock.json | |
| - name: Setup TypeScript client | |
| if: ${{ matrix.client == 'server' }} | |
| id: setup-ts-client | |
| uses: ./.github/actions/setup-typescript-client | |
| with: | |
| client-version: ${{ matrix.client-version }} | |
| - name: Run tests | |
| if: ${{ matrix.config.allowed_clients == null || contains(matrix.config.allowed_clients, matrix.client) }} | |
| uses: ./.github/actions/run-and-record-tests | |
| env: | |
| OPENAI_API_KEY: dummy | |
| AWS_BEARER_TOKEN_BEDROCK: replay-mode-dummy-key | |
| AWS_DEFAULT_REGION: us-west-2 | |
| TS_CLIENT_PATH: ${{ steps.setup-ts-client.outputs.ts-client-path || '' }} | |
| with: | |
| stack-config: >- | |
| ${{ matrix.config.stack_config | |
| || (matrix.client == 'library' && 'ci-tests') | |
| || (matrix.client == 'server' && 'server:ci-tests') | |
| || 'docker:ci-tests' }} | |
| setup: ${{ matrix.config.setup }} | |
| inference-mode: ${{ matrix.config.inference_mode || 'replay' }} | |
| suite: ${{ matrix.config.suite }} | |
| target-branch: ${{ inputs.pr_head_ref || '' }} | |
| is-fork-pr: ${{ inputs.is_fork_pr && 'true' || (github.event.pull_request.head.repo.full_name != github.repository && 'true' || 'false') }} |