Run the integration test suite with Kubernetes authentication #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Auth Tests | |
| run-name: Run the integration test suite with Kubernetes authentication | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| pull_request: | |
| branches: | |
| - main | |
| - 'release-[0-9]+.[0-9]+.x' | |
| paths: | |
| - 'distributions/**' | |
| - 'src/llama_stack/**' | |
| - '!src/llama_stack_ui/**' | |
| - 'tests/integration/**' | |
| - 'uv.lock' | |
| - 'pyproject.toml' | |
| - 'requirements.txt' | |
| - '.github/workflows/integration-auth-tests.yml' # This workflow | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test-matrix: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| auth-provider: [oauth2_token] | |
| fail-fast: false # we want to run all tests regardless of failure | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| - name: Install dependencies | |
| uses: ./.github/actions/setup-runner | |
| - name: Install minikube | |
| if: ${{ matrix.auth-provider == 'kubernetes' }} | |
| uses: medyagh/setup-minikube@e9e035a86bbc3caea26a450bd4dbf9d0c453682e # v0.0.21 | |
| - name: Start minikube | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| minikube start | |
| kubectl get pods -A | |
| - name: Configure Kube Auth | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| kubectl create namespace llama-stack | |
| kubectl create serviceaccount llama-stack-auth -n llama-stack | |
| kubectl create token llama-stack-auth -n llama-stack > llama-stack-auth-token | |
| - name: Set Kubernetes Config | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| echo "KUBERNETES_API_SERVER_URL=$(kubectl get --raw /.well-known/openid-configuration| jq -r .jwks_uri)" >> $GITHUB_ENV | |
| echo "KUBERNETES_CA_CERT_PATH=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.certificate-authority}')" >> $GITHUB_ENV | |
| echo "KUBERNETES_ISSUER=$(kubectl get --raw /.well-known/openid-configuration| jq -r .issuer)" >> $GITHUB_ENV | |
| echo "KUBERNETES_AUDIENCE=$(kubectl create token llama-stack-auth -n llama-stack --duration=1h | cut -d. -f2 | base64 -d | jq -r '.aud[0]')" >> $GITHUB_ENV | |
| echo "TOKEN=$(cat llama-stack-auth-token)" >> $GITHUB_ENV | |
| - name: Set Kube Auth Config and run server | |
| env: | |
| INFERENCE_MODEL: "meta-llama/Llama-3.2-3B-Instruct" | |
| if: ${{ matrix.auth-provider == 'oauth2_token' }} | |
| run: | | |
| run_dir=$(mktemp -d) | |
| cat <<EOF > $run_dir/config.yaml | |
| version: '2' | |
| distro_name: kube | |
| apis: [] | |
| providers: {} | |
| storage: | |
| backends: | |
| kv_default: | |
| type: kv_sqlite | |
| db_path: $run_dir/kvstore.db | |
| sql_default: | |
| type: sql_sqlite | |
| db_path: $run_dir/sql_store.db | |
| stores: | |
| metadata: | |
| namespace: registry | |
| backend: kv_default | |
| inference: | |
| table_name: inference_store | |
| backend: sql_default | |
| conversations: | |
| table_name: openai_conversations | |
| backend: sql_default | |
| prompts: | |
| namespace: prompts | |
| backend: kv_default | |
| server: | |
| port: 8321 | |
| EOF | |
| yq eval '.server.auth.provider_config.type = "${{ matrix.auth-provider }}"' -i $run_dir/config.yaml | |
| yq eval '.server.auth.provider_config.tls_cafile = "${{ env.KUBERNETES_CA_CERT_PATH }}"' -i $run_dir/config.yaml | |
| yq eval '.server.auth.provider_config.issuer = "${{ env.KUBERNETES_ISSUER }}"' -i $run_dir/config.yaml | |
| yq eval '.server.auth.provider_config.audience = "${{ env.KUBERNETES_AUDIENCE }}"' -i $run_dir/config.yaml | |
| yq eval '.server.auth.provider_config.jwks.uri = "${{ env.KUBERNETES_API_SERVER_URL }}"' -i $run_dir/config.yaml | |
| yq eval '.server.auth.provider_config.jwks.token = "${{ env.TOKEN }}"' -i $run_dir/config.yaml | |
| cat $run_dir/config.yaml | |
| # avoid line breaks in the server log, especially because we grep it below. | |
| export LLAMA_STACK_LOG_WIDTH=200 | |
| nohup uv run llama stack run $run_dir/config.yaml > server.log 2>&1 & | |
| - name: Wait for Llama Stack server to be ready | |
| run: | | |
| echo "Waiting for Llama Stack server..." | |
| for i in {1..30}; do | |
| # Note: /v1/health does not require authentication | |
| if curl -s -L http://localhost:8321/v1/health | grep -q "OK"; then | |
| echo "Llama Stack server is up!" | |
| if grep -q "Enabling authentication with provider: ${{ matrix.auth-provider }}" server.log; then | |
| echo "Llama Stack server is configured to use ${{ matrix.auth-provider }} auth" | |
| exit 0 | |
| else | |
| echo "Llama Stack server is not configured to use ${{ matrix.auth-provider }} auth" | |
| cat server.log | |
| exit 1 | |
| fi | |
| fi | |
| sleep 1 | |
| done | |
| echo "Llama Stack server failed to start" | |
| cat server.log | |
| exit 1 | |
| - name: Test auth | |
| run: | | |
| # Function to test API endpoint with authentication | |
| # Usage: test_endpoint <curl_args> <user_token_file> <expected_status> [output_file] | |
| test_endpoint() { | |
| local curl_args="$1" | |
| local user_token_file=$2 | |
| local expected_status=$3 | |
| local output_file=${4:-/dev/null} | |
| local status | |
| local extra_curl_args=(-s -L -o "$output_file" -w "%{http_code}") | |
| if [ "$user_token_file" != "none" ]; then | |
| extra_curl_args+=(-H "Authorization: Bearer $(cat $user_token_file)") | |
| fi | |
| set -x | |
| status=$(curl $curl_args "${extra_curl_args[@]}") | |
| set +x | |
| if [ "$status" = "$expected_status" ]; then | |
| echo " ✓ Status: $status (expected $expected_status)" | |
| return 0 | |
| else | |
| echo " ✗ Status: $status (expected $expected_status)" | |
| exit 1 | |
| fi | |
| } | |
| echo "Testing /v1/version without token (should succeed)..." | |
| test_endpoint "http://127.0.0.1:8321/v1/version" "none" "200" || exit 1 | |
| echo "Testing /v1/providers without token (should fail with 401)..." | |
| test_endpoint "http://127.0.0.1:8321/v1/providers" "none" "401" || exit 1 | |
| echo "Testing /v1/providers with valid token (should succeed)..." | |
| test_endpoint "http://127.0.0.1:8321/v1/providers" "llama-stack-auth-token" "200" "providers.json" || exit 1 | |
| cat providers.json | jq . > /dev/null && echo " ✓ Valid JSON response" |