Run the integration test suite with Kubernetes authentication #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Auth Tests | |
| run-name: Run the integration test suite with Kubernetes authentication | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| pull_request: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| paths: | |
| - 'distributions/**' | |
| - 'src/llama_stack/**' | |
| - '!src/llama_stack_ui/**' | |
| - 'tests/integration/**' | |
| - 'uv.lock' | |
| - 'pyproject.toml' | |
| - 'requirements.txt' | |
| - '.github/workflows/integration-auth-tests.yml' # This workflow | |
| - 'scripts/integration-auth-tests.sh' | |
| merge_group: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test-matrix: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| auth-provider: [oauth2_token] | |
| fail-fast: false # we want to run all tests regardless of failure | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Install dependencies | |
| uses: ./.github/actions/setup-runner | |
| - name: Install minikube | |
| if: ${{ matrix.auth-provider == 'kubernetes' }} | |
| uses: medyagh/setup-minikube@e9e035a86bbc3caea26a450bd4dbf9d0c453682e # v0.0.21 | |
| - name: Start minikube | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| minikube start | |
| kubectl get pods -A | |
| - name: Configure Kube Auth | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| kubectl create namespace llama-stack | |
| for account in llama-stack-auth llama-stack-user1 llama-stack-user2; do | |
| kubectl create serviceaccount $account -n llama-stack | |
| kubectl create token $account -n llama-stack > $account-token | |
| done | |
| - name: Set Kubernetes Config | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| { | |
| echo "KUBERNETES_API_SERVER_URL=$(kubectl get --raw /.well-known/openid-configuration| jq -r .jwks_uri)" | |
| echo "KUBERNETES_CA_CERT_PATH=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.certificate-authority}')" | |
| echo "KUBERNETES_ISSUER=$(kubectl get --raw /.well-known/openid-configuration| jq -r .issuer)" | |
| echo "KUBERNETES_AUDIENCE=$(kubectl create token llama-stack-auth -n llama-stack --duration=1h | cut -d. -f2 | base64 -d | jq -r '.aud[0]')" | |
| echo "TOKEN=$(cat llama-stack-auth-token)" | |
| } >> "$GITHUB_ENV" | |
| - name: Set Kube Auth Config and run server | |
| env: | |
| INFERENCE_MODEL: "meta-llama/Llama-3.2-3B-Instruct" | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| run_dir=$(mktemp -d) | |
| cat <<EOF > "$run_dir/config.yaml" | |
| version: '2' | |
| image_name: kube | |
| distro_name: kube | |
| apis: | |
| - files | |
| - inference | |
| providers: | |
| files: | |
| - provider_id: builtin-files | |
| provider_type: inline::localfs | |
| config: | |
| storage_dir: $run_dir/files | |
| metadata_store: | |
| table_name: files_metadata | |
| backend: sql_default | |
| storage: | |
| backends: | |
| kv_default: | |
| type: kv_sqlite | |
| db_path: $run_dir/kvstore.db | |
| sql_default: | |
| type: sql_sqlite | |
| db_path: $run_dir/sql_store.db | |
| stores: | |
| metadata: | |
| namespace: registry | |
| backend: kv_default | |
| inference: | |
| table_name: inference_store | |
| backend: sql_default | |
| conversations: | |
| table_name: openai_conversations | |
| backend: sql_default | |
| prompts: | |
| namespace: prompts | |
| backend: kv_default | |
| server: | |
| port: 8321 | |
| auth: | |
| access_policy: | |
| - permit: | |
| actions: [read, delete] | |
| resource: sql_record::openai_files::* | |
| when: | |
| - user with system:serviceaccount:llama-stack:llama-stack-user1 in roles | |
| - user in owners roles | |
| description: User1 can read and delete their Files | |
| - permit: | |
| actions: [read] | |
| resource: regex:sql_record::openai_files::file-[a-f0-9-]+ | |
| when: | |
| - user in owners roles | |
| description: Owners can read their Files | |
| route_policy: | |
| - permit: | |
| paths: ["/v1/health", "/v1/version"] | |
| description: Public endpoints (no auth required) | |
| - permit: | |
| paths: regex:^/v1/(files|providers)(/.*)?$ | |
| description: Authenticated users can access files and providers endpoints (regex test) | |
| - forbid: | |
| paths: "*" | |
| description: Forbid all other routes by default | |
| EOF | |
| yq eval '.server.auth.provider_config.type = "${{ matrix.auth-provider }}"' -i "$run_dir/config.yaml" | |
| yq eval '.server.auth.provider_config.tls_cafile = "${{ env.KUBERNETES_CA_CERT_PATH }}"' -i "$run_dir/config.yaml" | |
| yq eval '.server.auth.provider_config.issuer = "${{ env.KUBERNETES_ISSUER }}"' -i "$run_dir/config.yaml" | |
| yq eval '.server.auth.provider_config.audience = "${{ env.KUBERNETES_AUDIENCE }}"' -i "$run_dir/config.yaml" | |
| yq eval '.server.auth.provider_config.jwks.uri = "${{ env.KUBERNETES_API_SERVER_URL }}"' -i "$run_dir/config.yaml" | |
| yq eval '.server.auth.provider_config.jwks.token = "${{ env.TOKEN }}"' -i "$run_dir/config.yaml" | |
| cat "$run_dir/config.yaml" | |
| # avoid line breaks in the server log, especially because we grep it below. | |
| export LLAMA_STACK_LOG_WIDTH=200 | |
| nohup uv run llama stack run "$run_dir/config.yaml" > server.log 2>&1 & | |
| - name: Wait for Llama Stack server to be ready | |
| run: | | |
| echo "Waiting for Llama Stack server..." | |
| for _ in {1..30}; do | |
| # Note: /v1/health does not require authentication | |
| if curl -s -L http://localhost:8321/v1/health | grep -q "OK"; then | |
| echo "Llama Stack server is up!" | |
| if grep -q "Enabling authentication with provider: ${{ matrix.auth-provider }}" server.log; then | |
| echo "Llama Stack server is configured to use ${{ matrix.auth-provider }} auth" | |
| exit 0 | |
| else | |
| echo "Llama Stack server is not configured to use ${{ matrix.auth-provider }} auth" | |
| cat server.log | |
| exit 1 | |
| fi | |
| fi | |
| sleep 1 | |
| done | |
| echo "Llama Stack server failed to start" | |
| cat server.log | |
| exit 1 | |
| - name: Test auth | |
| run: | | |
| # Run the auth tests | |
| ./scripts/integration-auth-tests.sh |