diff --git a/.github/workflows/nightly-parallel.yml b/.github/workflows/nightly-parallel.yml new file mode 100644 index 000000000..8c25d4359 --- /dev/null +++ b/.github/workflows/nightly-parallel.yml @@ -0,0 +1,490 @@ +name: Nightly Parallel Integration Tests + +on: + schedule: + - cron: '30 18 * * *' # 12:00 AM IST (UTC+5:30 = 18:30 UTC) + workflow_dispatch: + inputs: + reason: + description: 'Reason for manual trigger' + required: false + default: 'Manual run' + +concurrency: + group: nightly-parallel + cancel-in-progress: true + +permissions: + contents: read + +env: + GO_COVERAGE_PKGS: "github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers" + +jobs: + # ───────────────────────────────────────────────────────────────────────────── + # Job A: Scan all integration test files and detect any tests not yet assigned + # to a named matrix group, so they fall through to the catch-all run. + # ───────────────────────────────────────────────────────────────────────────── + validate-test-coverage: + runs-on: ubuntu-latest + outputs: + uncovered_tests: ${{ steps.find-uncovered.outputs.uncovered_tests }} + has_uncovered: ${{ steps.find-uncovered.outputs.has_uncovered }} + steps: + - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + + - name: Find tests not covered by any named group + id: find-uncovered + run: | + # Combined regex of every pattern used across the 12 named matrix groups. + # Any test whose name does NOT match this will land in the catch-all group. + # Built via concatenation so every line stays at ≥10-space YAML indentation. + CP="TestCreateScan|TestScanCreate|TestScansE2E|TestFastScan" + CP="${CP}|TestLightQueries|TestRecommendedExclusions|TestScansUpdateProjectGroups" + CP="${CP}|TestBrokenLinkScan|TestScanWithPolicy|TestCreateAsyncScan|TestScanGLReport" + CP="${CP}|TestContainerEngineScansE2E|TestScanListWith|TestScanShowRequired" + CP="${CP}|TestRequiredScanId|TestScaResolver|TestInvalidSource|TestIncrementalScan" + CP="${CP}|TestBranchPrimary|TestCancelScan|TestScanTimeout|TestScanWorkflow|TestScanLog" + CP="${CP}|TestPartialScan|TestFailedScan|TestRunKics|TestRunSca|TestScaRealtime" + CP="${CP}|TestScanType|TestValidateScan|TestScanGenerating|TestResult|TestCodeBashing" + CP="${CP}|TestRiskManagement|TestCreateQueryDescription|TestPR|TestPreReceive" + CP="${CP}|TestPre_Receive|TestProject|TestCreateEmptyProject|TestCreateAlreadyExisting" + CP="${CP}|TestCreateWithInvalid|TestCreateProjectWhen|TestGetProject|TestSastUpdate" + CP="${CP}|TestGetAndUpdate|TestPredicate|TestTriage|TestScaUpdate|TestRunGetBfl" + CP="${CP}|TestContainerScan|TestContainerImage|TestContainersRealtime|TestIacRealtime" + CP="${CP}|TestSecrets_Realtime|TestOssRealtime|TestScanASCA|TestExecuteASCA" + CP="${CP}|TestEngineNameResolution|TestAuth|TestFailProxy|TestLoadConfiguration" + CP="${CP}|TestSetConfig|TestMain|TestRootVersion|TestSetLog|Test_Download|TestGitHub" + CP="${CP}|TestGitLab|TestBitbucket|TestBitBucket|TestAzure|TestHooksPreCommit" + CP="${CP}|TestGetLearnMore|TestImport|TestGetTenant|TestMaskSecrets|TestFailedMask" + CP="${CP}|TestScaRemediation|TestKicsRemediation|TestTelemetry|Test_Handle|TestChat" + COVERED_PATTERNS="${CP}" + + ALL_TESTS=$(grep -rh "^func Test" test/integration/*_test.go \ + | sed 's/func \(Test[^(]*\).*/\1/') + + UNCOVERED="" + while IFS= read -r test; do + [ -z "$test" ] && continue + if ! echo "$test" | grep -qE "$COVERED_PATTERNS"; then + UNCOVERED="${UNCOVERED:+${UNCOVERED}|}${test}" + fi + done <<< "$ALL_TESTS" + + if [ -n "$UNCOVERED" ]; then + echo "Uncovered tests detected: $UNCOVERED" + { + echo "uncovered_tests=$UNCOVERED" + echo "has_uncovered=true" + } >> "$GITHUB_OUTPUT" + else + echo "All tests are covered by named groups." + { + echo "uncovered_tests=" + echo "has_uncovered=false" + } >> "$GITHUB_OUTPUT" + fi + + # ───────────────────────────────────────────────────────────────────────────── + # Job B: Run each test group in parallel across 13 matrix entries. + # The 13th entry (uncovered) is a dynamic catch-all driven by Job A. + # ───────────────────────────────────────────────────────────────────────────── + integration-tests: + needs: validate-test-coverage + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # 1 ── Scan Creation (heavy; needs pre-run cleanup) + - name: scan-create + label: "Scan Creation" + run_pattern: "TestCreateScan|TestScanCreate|TestScansE2E|TestFastScan|TestLightQueries|TestRecommendedExclusions|TestScansUpdateProjectGroups|TestBrokenLinkScan|TestScanWithPolicy|TestCreateAsyncScan|TestScanGLReport|TestContainerEngineScansE2E" + timeout: "90m" + needs_precommit: "false" + run_cleandata: "true" + + # 2 ── Scan Operations (list, show, logs, kics, sca; needs pre-run cleanup) + - name: scan-ops + label: "Scan Operations" + run_pattern: "TestScanListWith|TestScanShowRequired|TestRequiredScanId|TestScaResolver|TestInvalidSource|TestIncrementalScan|TestBranchPrimary|TestCancelScan|TestScanTimeout|TestScanWorkflow|TestScanLog|TestPartialScan|TestFailedScan|TestRunKics|TestRunSca|TestScaRealtime|TestScanType|TestValidateScan|TestScanGenerating" + timeout: "90m" + needs_precommit: "false" + run_cleandata: "true" + + # 3 ── Results & Reports + - name: results + label: "Results & Reports" + run_pattern: "TestResult|TestCodeBashing|TestRiskManagement|TestCreateQueryDescription" + timeout: "60m" + needs_precommit: "false" + run_cleandata: "false" + + # 4 ── PR Decoration + - name: pr-decoration + label: "PR Decoration" + run_pattern: "TestPR|TestPreReceive|TestPre_Receive" + timeout: "60m" + needs_precommit: "false" + run_cleandata: "false" + + # 5 ── Projects (needs pre-run cleanup) + - name: projects + label: "Projects" + run_pattern: "TestProject|TestCreateEmptyProject|TestCreateAlreadyExisting|TestCreateWithInvalid|TestCreateProjectWhen|TestGetProject" + timeout: "60m" + needs_precommit: "false" + run_cleandata: "true" + + # 6 ── Predicates & BFL + - name: predicates + label: "Predicates & BFL" + run_pattern: "TestSastUpdate|TestGetAndUpdate|TestPredicate|TestTriage|TestScaUpdate|TestRunGetBfl" + timeout: "60m" + needs_precommit: "false" + run_cleandata: "false" + + # 7 ── Container Tests + - name: containers + label: "Container Tests" + run_pattern: "TestContainerScan|TestContainerImage" + timeout: "60m" + needs_precommit: "false" + run_cleandata: "false" + + # 8 ── Realtime Scanning (ASCA, IAC, Secrets, OSS) + - name: realtime + label: "Realtime Scanning" + run_pattern: "TestContainersRealtime|TestIacRealtime|TestSecrets_Realtime|TestOssRealtime|TestScanASCA|TestExecuteASCA|TestEngineNameResolution" + timeout: "60m" + needs_precommit: "false" + run_cleandata: "false" + + # 9 ── Auth & Config + - name: auth-config + label: "Auth & Config" + run_pattern: "TestAuth|TestFailProxy|TestLoadConfiguration|TestSetConfig" + timeout: "30m" + needs_precommit: "false" + run_cleandata: "false" + + # 10 ── Root & Logs + - name: root-logs + label: "Root & Logs" + run_pattern: "TestMain|TestRootVersion|TestSetLog|Test_Download" + timeout: "30m" + needs_precommit: "false" + run_cleandata: "false" + + # 11 ── SCM Rate Limit & User Count + - name: scm-tests + label: "SCM Rate Limit & User Count" + run_pattern: "TestGitHub|TestGitLab|TestBitbucket|TestBitBucket|TestAzure" + timeout: "60m" + needs_precommit: "false" + run_cleandata: "false" + + # 12 ── Miscellaneous (pre-commit required) + - name: misc + label: "Miscellaneous" + run_pattern: "TestHooksPreCommit|TestGetLearnMore|TestImport|TestGetTenant|TestMaskSecrets|TestFailedMask|TestScaRemediation|TestKicsRemediation|TestTelemetry|Test_Handle|TestChat" + timeout: "60m" + needs_precommit: "true" + run_cleandata: "false" + + # 13 ── Catch-All (dynamic; pattern injected at runtime from Job A output) + - name: uncovered + label: "Catch-All (Uncovered)" + run_pattern: "__UNCOVERED__" + timeout: "90m" + needs_precommit: "false" + run_cleandata: "false" + + env: + CX_BASE_URI: ${{ secrets.CX_BASE_URI }} + CX_CLIENT_ID: ${{ secrets.CX_CLIENT_ID }} + CX_CLIENT_SECRET: ${{ secrets.CX_CLIENT_SECRET }} + CX_BASE_AUTH_URI: ${{ secrets.CX_BASE_AUTH_URI }} + CX_AST_USERNAME: ${{ secrets.CX_AST_USERNAME }} + CX_AST_PASSWORD: ${{ secrets.CX_AST_PASSWORD }} + CX_APIKEY: ${{ secrets.CX_APIKEY }} + CX_TENANT: ${{ secrets.CX_TENANT }} + CX_SCAN_SSH_KEY: ${{ secrets.CX_SCAN_SSH_KEY }} + CX_ORIGIN: "cli-tests" + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + PROXY_HOST: localhost + PROXY_PORT: 3128 + PROXY_USERNAME: ${{ secrets.PROXY_USER }} + PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }} + PR_GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + PR_GITHUB_NAMESPACE: "checkmarx" + PR_GITHUB_REPO_NAME: "ast-cli" + PR_GITHUB_NUMBER: 983 + PR_GITLAB_TOKEN: ${{ secrets.PR_GITLAB_TOKEN }} + PR_GITLAB_NAMESPACE: ${{ secrets.PR_GITLAB_NAMESPACE }} + PR_GITLAB_REPO_NAME: ${{ secrets.PR_GITLAB_REPO_NAME }} + PR_GITLAB_PROJECT_ID: ${{ secrets.PR_GITLAB_PROJECT_ID }} + PR_GITLAB_IID: ${{ secrets.PR_GITLAB_IID }} + AZURE_ORG: ${{ secrets.AZURE_ORG }} + AZURE_PROJECT: ${{ secrets.AZURE_PROJECT }} + AZURE_REPOS: ${{ secrets.AZURE_REPOS }} + AZURE_TOKEN: ${{ secrets.AZURE_TOKEN }} + AZURE_PR_NUMBER: 1 + BITBUCKET_WORKSPACE: ${{ secrets.BITBUCKET_WORKSPACE }} + BITBUCKET_REPOS: ${{ secrets.BITBUCKET_REPOS }} + BITBUCKET_USERNAME: ${{ secrets.BITBUCKET_USERNAME }} + BITBUCKET_PASSWORD: ${{ secrets.BITBUCKET_PASSWORD }} + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} + PR_BITBUCKET_TOKEN: ${{ secrets.PR_BITBUCKET_TOKEN }} + PR_BITBUCKET_NAMESPACE: "AstSystemTest" + PR_BITBUCKET_REPO_NAME: "cliIntegrationTest" + PR_BITBUCKET_ID: 1 + + steps: + - name: Checkout repository + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + + - name: Set up Go + uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 + with: + go-version-file: go.mod + + - name: Build binary + run: go build -o ./bin/cx ./cmd + + - name: Install gocovmerge + run: go install github.com/wadey/gocovmerge@latest + + - name: Install pre-commit + if: matrix.needs_precommit == 'true' + run: | + pip install pre-commit + pre-commit install + + - name: Start Squid proxy + run: | + docker run \ + --name squid \ + -d \ + -p 3128:3128 \ + -v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \ + -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \ + ubuntu/squid:5.2-22.04_beta + + - name: Download ScaResolver + run: | + wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz + tar -xzvf ScaResolver-linux64.tar.gz -C /tmp + rm -rf ScaResolver-linux64.tar.gz + + - name: Pre-test cleanup (${{ matrix.label }}) + if: matrix.run_cleandata == 'true' + run: go test -v github.com/checkmarx/ast-cli/test/cleandata + + - name: Run integration tests (${{ matrix.label }}) + if: matrix.name != 'uncovered' || needs.validate-test-coverage.outputs.has_uncovered == 'true' + run: | + set -euo pipefail + + # Resolve the -run pattern: catch-all uses Job A output; named groups use matrix field. + if [ "${{ matrix.name }}" = "uncovered" ]; then + RUN_PATTERN="${{ needs.validate-test-coverage.outputs.uncovered_tests }}" + else + RUN_PATTERN="${{ matrix.run_pattern }}" + fi + + COVER_FILE="cover-${{ matrix.name }}.out" + + run_tests() { + local pattern="$1" outfile="$2" logfile="$3" timeout_val="$4" + go test \ + -tags integration \ + -v \ + -timeout "${timeout_val}" \ + -coverpkg "${{ env.GO_COVERAGE_PKGS }}" \ + -coverprofile "${outfile}" \ + -run "${pattern}" \ + github.com/checkmarx/ast-cli/test/integration 2>&1 | tee "${logfile}" || true + } + + echo "::group::Attempt 1 — ${{ matrix.label }}" + run_tests "$RUN_PATTERN" "$COVER_FILE" "test_output.log" "${{ matrix.timeout }}" + echo "::endgroup::" + + FAILED=$(grep -E "^--- FAIL: " test_output.log | awk '{print $3}' | paste -sd '|' - || true) + + # ── Retry 1 ────────────────────────────────────────────────────────── + if [ -n "$FAILED" ]; then + echo "::warning::Retry 1 for ${{ matrix.label }}: $FAILED" + COVER_R1="cover-${{ matrix.name }}-r1.out" + echo "::group::Attempt 2 — ${{ matrix.label }}" + run_tests "$FAILED" "$COVER_R1" "retry1_output.log" "30m" + echo "::endgroup::" + + if [ -f "$COVER_R1" ]; then + gocovmerge "$COVER_FILE" "$COVER_R1" > merged.out + mv merged.out "$COVER_FILE" + rm -f "$COVER_R1" + fi + + FAILED2=$(grep -E "^--- FAIL: " retry1_output.log | awk '{print $3}' | paste -sd '|' - || true) + + # ── Retry 2 ──────────────────────────────────────────────────────── + if [ -n "$FAILED2" ]; then + echo "::warning::Retry 2 for ${{ matrix.label }}: $FAILED2" + COVER_R2="cover-${{ matrix.name }}-r2.out" + echo "::group::Attempt 3 — ${{ matrix.label }}" + run_tests "$FAILED2" "$COVER_R2" "retry2_output.log" "30m" + echo "::endgroup::" + + if [ -f "$COVER_R2" ]; then + gocovmerge "$COVER_FILE" "$COVER_R2" > merged.out + mv merged.out "$COVER_FILE" + rm -f "$COVER_R2" + fi + + FINAL_FAILED=$(grep -E "^--- FAIL: " retry2_output.log | awk '{print $3}' || true) + if [ -n "$FINAL_FAILED" ]; then + echo "::error::Tests still failing after 2 retries in ${{ matrix.label }}: $FINAL_FAILED" + exit 1 + fi + fi + fi + + echo "All ${{ matrix.label }} tests passed." + + - name: Upload coverage artifact + if: always() && (matrix.name != 'uncovered' || needs.validate-test-coverage.outputs.has_uncovered == 'true') + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + with: + name: coverage-${{ matrix.name }} + path: cover-${{ matrix.name }}.out + retention-days: 7 + if-no-files-found: warn + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + with: + name: test-logs-${{ matrix.name }} + path: | + test_output.log + retry1_output.log + retry2_output.log + retention-days: 7 + if-no-files-found: ignore + + - name: Stop Squid proxy + if: always() + run: docker stop squid && docker rm squid || true + + # ───────────────────────────────────────────────────────────────────────────── + # Job C: Download all per-group coverage files, merge them, check >= 75%, + # upload the HTML report, and run a final project cleanup. + # ───────────────────────────────────────────────────────────────────────────── + merge-coverage: + needs: integration-tests + runs-on: ubuntu-latest + if: always() + env: + CX_BASE_URI: ${{ secrets.CX_BASE_URI }} + CX_CLIENT_ID: ${{ secrets.CX_CLIENT_ID }} + CX_CLIENT_SECRET: ${{ secrets.CX_CLIENT_SECRET }} + CX_BASE_AUTH_URI: ${{ secrets.CX_BASE_AUTH_URI }} + CX_AST_USERNAME: ${{ secrets.CX_AST_USERNAME }} + CX_AST_PASSWORD: ${{ secrets.CX_AST_PASSWORD }} + CX_APIKEY: ${{ secrets.CX_APIKEY }} + CX_TENANT: ${{ secrets.CX_TENANT }} + steps: + - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + + - name: Set up Go + uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 + with: + go-version-file: go.mod + + - name: Install gocovmerge + run: go install github.com/wadey/gocovmerge@latest + + - name: Download all coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: coverage-* + merge-multiple: true + + - name: Merge coverage profiles + run: | + COVER_FILES=$(ls cover-*.out 2>/dev/null | tr '\n' ' ') + if [ -z "$COVER_FILES" ]; then + echo "::error::No coverage files found — all groups may have been skipped or failed." + exit 1 + fi + echo "Merging: $COVER_FILES" + # shellcheck disable=SC2086 + gocovmerge $COVER_FILES > cover.out + go tool cover -html=cover.out -o coverage.html + + - name: Check coverage threshold (>= 75%) + run: | + CODE_COV=$(go tool cover -func cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}') + EXPECTED=75 + echo "Total coverage: ${CODE_COV}%" + var=$(awk 'BEGIN{ print "'$CODE_COV'"<"'$EXPECTED'" }') + if [ "$var" -eq 1 ]; then + echo "::error::Coverage too low: ${CODE_COV}% (required >= ${EXPECTED}%)" + exit 1 + else + echo "Coverage OK: ${CODE_COV}%" + fi + + - name: Upload merged HTML report + if: always() + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + with: + name: coverage-report-merged + path: coverage.html + retention-days: 7 + + - name: Post-run project cleanup + run: go test -v github.com/checkmarx/ast-cli/test/cleandata + + # ───────────────────────────────────────────────────────────────────────────── + # Job D: Write a GitHub Actions Job Summary when any job in the chain fails. + # No external service required — everything goes to GITHUB_STEP_SUMMARY. + # ───────────────────────────────────────────────────────────────────────────── + notify-on-failure: + needs: [integration-tests, merge-coverage] + runs-on: ubuntu-latest + if: failure() + steps: + - name: Write failure summary + env: + INTEGRATION_RESULT: ${{ toJson(needs.integration-tests) }} + MERGE_RESULT: ${{ toJson(needs.merge-coverage) }} + run: | + cat >> "$GITHUB_STEP_SUMMARY" << 'SUMMARY' + ## Nightly Parallel Integration Tests — FAILED + + | Field | Value | + |-------|-------| + | Run | ${{ github.run_id }} | + | Triggered by | ${{ github.event_name }} | + | Branch | ${{ github.ref_name }} | + | Commit | ${{ github.sha }} | + | Schedule | 12:00 AM IST (18:30 UTC) | + SUMMARY + + printf '\n### integration-tests result\n```json\n%s\n```\n' "$INTEGRATION_RESULT" \ + >> "$GITHUB_STEP_SUMMARY" + printf '\n### merge-coverage result\n```json\n%s\n```\n' "$MERGE_RESULT" \ + >> "$GITHUB_STEP_SUMMARY" + + cat >> "$GITHUB_STEP_SUMMARY" << 'SUMMARY' + + ### Next Steps + 1. Click each failed matrix group in the job list above to inspect its logs. + 2. Download the `test-logs-` artifact for the full `go test` output. + 3. Retry a specific group manually via **Run workflow** (`workflow_dispatch`). + 4. If the failure is consistent, open an issue referencing this run. + SUMMARY