Add integration test setup, sharding, and validation scripts(AST-2323) #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous Integration Tests (Sharded) | |
| on: | |
| pull_request: | |
| # Ensure we don't have multiple runs for the same PR | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| TOTAL_SHARDS: 4 | |
| EXPECTED_COVERAGE: 75 | |
| jobs: | |
| # ========================================================================== | |
| # Unit Tests (unchanged) | |
| # ========================================================================== | |
| unit-tests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 | |
| - name: Set up Go version | |
| uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 | |
| with: | |
| go-version-file: go.mod | |
| cache: true | |
| - run: go version | |
| - name: go test with coverage | |
| run: | | |
| sudo chmod +x ./internal/commands/.scripts/up.sh | |
| ./internal/commands/.scripts/up.sh | |
| - name: Check if total coverage is greater then 77.7 | |
| shell: bash | |
| run: | | |
| CODE_COV=$(go tool cover -func cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}') | |
| EXPECTED_CODE_COV=77.7 | |
| var=$(awk 'BEGIN{ print "'$CODE_COV'"<"'$EXPECTED_CODE_COV'" }') | |
| if [ "$var" -eq 1 ];then | |
| echo "Your code coverage is too low. Coverage precentage is: $CODE_COV" | |
| exit 1 | |
| else | |
| echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" | |
| exit 0 | |
| fi | |
| # ========================================================================== | |
| # Integration Tests - Sharded Execution | |
| # ========================================================================== | |
| integration-tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false # Run all shards even if one fails | |
| max-parallel: 4 | |
| matrix: | |
| shard: [1, 2, 3, 4] | |
| outputs: | |
| shard-1-status: ${{ steps.run-tests.outputs.status }} | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 | |
| - name: Set up Go version | |
| uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 | |
| with: | |
| go-version-file: go.mod | |
| cache: true | |
| - run: go version | |
| - name: Go Build | |
| run: go build -o ./bin/cx ./cmd | |
| - name: Install gocovmerge | |
| run: go install github.com/wadey/gocovmerge@latest | |
| - name: Install jq (for manifest parsing) | |
| run: sudo apt-get install -y jq | |
| - name: Install pre-commit | |
| run: | | |
| pip install pre-commit | |
| pre-commit install | |
| # Cache ScaResolver to avoid downloading on every run | |
| - name: Cache ScaResolver | |
| id: cache-scaresolver | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/ScaResolver | |
| key: scaresolver-linux64-v1 | |
| restore-keys: | | |
| scaresolver-linux64- | |
| - name: Download ScaResolver | |
| if: steps.cache-scaresolver.outputs.cache-hit != 'true' | |
| run: | | |
| wget -q https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz | |
| tar -xzf ScaResolver-linux64.tar.gz -C /tmp | |
| rm -f ScaResolver-linux64.tar.gz | |
| # Cache Docker image for Squid proxy | |
| - name: Pull and cache Squid image | |
| run: docker pull ubuntu/squid:5.2-22.04_beta | |
| - name: Start Squid Proxy | |
| run: | | |
| docker run \ | |
| --name squid \ | |
| -d \ | |
| -p ${{ env.PROXY_PORT }}:3128 \ | |
| -v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \ | |
| -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \ | |
| ubuntu/squid:5.2-22.04_beta | |
| env: | |
| PROXY_PORT: 3128 | |
| - name: Run Integration Tests (Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}) | |
| id: run-tests | |
| shell: bash | |
| env: | |
| # Shard configuration | |
| SHARD_INDEX: ${{ matrix.shard }} | |
| TOTAL_SHARDS: ${{ env.TOTAL_SHARDS }} | |
| TEST_TIMEOUT: 45m | |
| RERUN_TIMEOUT: 15m | |
| # Checkmarx credentials | |
| CX_BASE_URI: ${{ secrets.CX_BASE_URI }} | |
| CX_CLIENT_ID: ${{ secrets.CX_CLIENT_ID }} | |
| CX_CLIENT_SECRET: ${{ secrets.CX_CLIENT_SECRET }} | |
| CX_BASE_AUTH_URI: ${{ secrets.CX_BASE_AUTH_URI }} | |
| CX_AST_USERNAME: ${{ secrets.CX_AST_USERNAME }} | |
| CX_AST_PASSWORD: ${{ secrets.CX_AST_PASSWORD }} | |
| CX_APIKEY: ${{ secrets.CX_APIKEY }} | |
| CX_TENANT: ${{ secrets.CX_TENANT }} | |
| CX_SCAN_SSH_KEY: ${{ secrets.CX_SCAN_SSH_KEY }} | |
| CX_ORIGIN: "cli-tests" | |
| # Proxy settings | |
| PROXY_HOST: localhost | |
| PROXY_PORT: 3128 | |
| PROXY_USERNAME: ${{ secrets.PROXY_USER }} | |
| PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }} | |
| # GitHub PR settings | |
| PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| PR_GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| PR_GITHUB_NAMESPACE: "checkmarx" | |
| PR_GITHUB_REPO_NAME: "ast-cli" | |
| PR_GITHUB_NUMBER: 983 | |
| # GitLab settings | |
| PR_GITLAB_TOKEN: ${{ secrets.PR_GITLAB_TOKEN }} | |
| PR_GITLAB_NAMESPACE: ${{ secrets.PR_GITLAB_NAMESPACE }} | |
| PR_GITLAB_REPO_NAME: ${{ secrets.PR_GITLAB_REPO_NAME }} | |
| PR_GITLAB_PROJECT_ID: ${{ secrets.PR_GITLAB_PROJECT_ID }} | |
| PR_GITLAB_IID: ${{ secrets.PR_GITLAB_IID }} | |
| GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} | |
| # Azure settings | |
| AZURE_ORG: ${{ secrets.AZURE_ORG }} | |
| AZURE_PROJECT: ${{ secrets.AZURE_PROJECT }} | |
| AZURE_REPOS: ${{ secrets.AZURE_REPOS }} | |
| AZURE_TOKEN: ${{ secrets.AZURE_TOKEN }} | |
| AZURE_NEW_ORG: ${{ secrets.AZURE_NEW_ORG }} | |
| AZURE_PROJECT_NAME: ${{ secrets.AZURE_PROJECT_NAME }} | |
| AZURE_PR_NUMBER: 1 | |
| AZURE_NEW_TOKEN: ${{ secrets.AZURE_NEW_TOKEN }} | |
| # Bitbucket settings | |
| BITBUCKET_WORKSPACE: ${{ secrets.BITBUCKET_WORKSPACE }} | |
| BITBUCKET_REPOS: ${{ secrets.BITBUCKET_REPOS }} | |
| BITBUCKET_USERNAME: ${{ secrets.BITBUCKET_USERNAME }} | |
| BITBUCKET_PASSWORD: ${{ secrets.BITBUCKET_PASSWORD }} | |
| PR_BITBUCKET_TOKEN: ${{ secrets.PR_BITBUCKET_TOKEN }} | |
| PR_BITBUCKET_NAMESPACE: "AstSystemTest" | |
| PR_BITBUCKET_REPO_NAME: "cliIntegrationTest" | |
| PR_BITBUCKET_ID: 1 | |
| GITHUB_ACTOR: ${{ github.actor }} | |
| run: | | |
| chmod +x ./internal/commands/.scripts/integration_shard.sh | |
| # Run the shard | |
| set +e | |
| ./internal/commands/.scripts/integration_shard.sh ${{ matrix.shard }} ${{ env.TOTAL_SHARDS }} | |
| exit_code=$? | |
| set -e | |
| # Set output for downstream jobs | |
| if [ $exit_code -eq 0 ]; then | |
| echo "status=passed" >> $GITHUB_OUTPUT | |
| else | |
| echo "status=failed" >> $GITHUB_OUTPUT | |
| fi | |
| exit $exit_code | |
| - name: Stop Squid Proxy | |
| if: always() | |
| run: docker rm -f squid || true | |
| - name: Upload Shard Artifacts | |
| if: always() | |
| uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 | |
| with: | |
| name: shard-${{ matrix.shard }}-results | |
| path: | | |
| shard_${{ matrix.shard }}_of_${{ env.TOTAL_SHARDS }}/ | |
| retention-days: 7 | |
| # ========================================================================== | |
| # Validate All Tests Were Executed | |
| # ========================================================================== | |
| validate-integration-tests: | |
| runs-on: ubuntu-latest | |
| needs: integration-tests | |
| if: always() # Run even if some shards failed | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 | |
| - name: Set up Go version | |
| uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 | |
| with: | |
| go-version-file: go.mod | |
| cache: true | |
| - name: Install gocovmerge | |
| run: go install github.com/wadey/gocovmerge@latest | |
| - name: Install jq | |
| run: sudo apt-get install -y jq | |
| - name: Download All Shard Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: shard-*-results | |
| merge-multiple: false | |
| - name: Organize Artifacts | |
| run: | | |
| # Move artifacts to expected structure | |
| for shard in $(seq 1 ${{ env.TOTAL_SHARDS }}); do | |
| artifact_dir="shard-${shard}-results" | |
| if [ -d "$artifact_dir" ]; then | |
| # The artifact might contain nested directory | |
| if [ -d "$artifact_dir/shard_${shard}_of_${{ env.TOTAL_SHARDS }}" ]; then | |
| mv "$artifact_dir/shard_${shard}_of_${{ env.TOTAL_SHARDS }}" . | |
| else | |
| mv "$artifact_dir" "shard_${shard}_of_${{ env.TOTAL_SHARDS }}" | |
| fi | |
| fi | |
| done | |
| # Debug: List what we have | |
| echo "Available shard directories:" | |
| ls -la shard_* || echo "No shard directories found" | |
| - name: Validate All Tests Executed | |
| id: validate | |
| env: | |
| TOTAL_SHARDS: ${{ env.TOTAL_SHARDS }} | |
| EXPECTED_COVERAGE: ${{ env.EXPECTED_COVERAGE }} | |
| ARTIFACTS_DIR: "." | |
| run: | | |
| chmod +x ./internal/commands/.scripts/integration_validate.sh | |
| set +e | |
| ./internal/commands/.scripts/integration_validate.sh ${{ env.TOTAL_SHARDS }} | |
| exit_code=$? | |
| set -e | |
| if [ $exit_code -eq 0 ]; then | |
| echo "validation=passed" >> $GITHUB_OUTPUT | |
| else | |
| echo "validation=failed" >> $GITHUB_OUTPUT | |
| fi | |
| exit $exit_code | |
| - name: Upload Validation Report | |
| if: always() | |
| uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 | |
| with: | |
| name: validation-report | |
| path: | | |
| validation_report.json | |
| test_summary.md | |
| merged_coverage.out | |
| coverage.html | |
| retention-days: 30 | |
| - name: Upload Final Coverage | |
| uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 | |
| with: | |
| name: ${{ runner.os }}-coverage-latest | |
| path: coverage.html | |
| - name: Post Summary to PR | |
| if: always() | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| let summary = '## Integration Test Results\n\n'; | |
| try { | |
| if (fs.existsSync('test_summary.md')) { | |
| summary = fs.readFileSync('test_summary.md', 'utf8'); | |
| } | |
| } catch (e) { | |
| summary += 'Unable to read test summary.\n'; | |
| } | |
| // Also add to job summary | |
| await core.summary.addRaw(summary).write(); | |
| - name: Check Validation Status | |
| if: always() | |
| run: | | |
| if [ -f validation_report.json ]; then | |
| echo "=== Validation Report ===" | |
| cat validation_report.json | jq . | |
| status=$(jq -r '.final_status' validation_report.json) | |
| if [ "$status" != "passed" ]; then | |
| echo "::error::Integration tests validation failed!" | |
| exit 1 | |
| fi | |
| else | |
| echo "::error::Validation report not found!" | |
| exit 1 | |
| fi | |
| # ========================================================================== | |
| # Lint (unchanged) | |
| # ========================================================================== | |
| lint: | |
| name: lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 | |
| - name: Set up Go version | |
| uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 | |
| with: | |
| go-version-file: go.mod | |
| cache: true | |
| - run: go version | |
| - run: go mod tidy | |
| - name: golangci-lint | |
| uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc #v3 | |
| with: | |
| skip-pkg-cache: true | |
| version: v1.64.2 | |
| args: -c .golangci.yml --timeout 5m | |
| only-new-issues: true | |
| # ========================================================================== | |
| # Security Scans (unchanged) | |
| # ========================================================================== | |
| govulncheck: | |
| runs-on: ubuntu-latest | |
| name: govulncheck | |
| steps: | |
| - id: govulncheck | |
| uses: golang/govulncheck-action@7da72f730e37eeaad891fcff0a532d27ed737cd4 #v1 | |
| continue-on-error: true | |
| with: | |
| go-version-file: go.mod | |
| go-package: ./... | |
| checkDockerImage: | |
| runs-on: ubuntu-latest | |
| name: scan Docker Image with Trivy | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@722adc63f1aa60a57ec37892e133b1d319cae598 #2.0.0 | |
| - name: Set up Docker | |
| uses: docker/setup-buildx-action@cf09c5c41b299b55c366aff30022701412eb6ab0 #v1.0.0 | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b #v2 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Build the project | |
| run: go build -o ./cx ./cmd | |
| - name: Build Docker image | |
| run: docker build -t ast-cli:${{ github.sha }} . | |
| - name: Run Trivy scanner without downloading DBs | |
| uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 #v0.28.0 | |
| with: | |
| scan-type: 'image' | |
| image-ref: ast-cli:${{ github.sha }} | |
| format: 'table' | |
| exit-code: '1' | |
| ignore-unfixed: true | |
| vuln-type: 'os,library' | |
| output: './trivy-image-results.txt' | |
| env: | |
| TRIVY_SKIP_JAVA_DB_UPDATE: true | |
| - name: Inspect action report | |
| if: always() | |
| shell: bash | |
| run: cat ./trivy-image-results.txt |