diff --git a/.github/workflows/test_coverage.yml b/.github/workflows/test_coverage.yml new file mode 100644 index 0000000000..e5c713d049 --- /dev/null +++ b/.github/workflows/test_coverage.yml @@ -0,0 +1,182 @@ +name: Test coverage +on: + pull_request: + # Run manually. + workflow_dispatch: {} + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch + # every day at 00:00 UTC. + schedule: + - cron: '0 0 * * *' + +env: + CSFY_CI: true + +# Set up permissions for OIDC authentication. +permissions: + # This is required for requesting the OIDC JWT. + id-token: write + # This is required for actions/checkout. + contents: read + # This is required for pulling the Docker image from GHCR. + packages: read +jobs: + run_test_coverage: + runs-on: ubuntu-latest + + steps: + # Configure AWS authentication for this workflow. + # This step assumes an AWS IAM role to grant GH Action temporary + # credentials necessary to access AWS resources. + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ vars.GH_ACTION_AWS_ROLE_ARN }} + role-session-name: ${{ vars.GH_ACTION_AWS_SESSION_NAME }} + aws-region: ${{ vars.CSFY_AWS_DEFAULT_REGION }} + + # This is needed to pull the Docker image. + - name: Login to GHCR + run: docker login ghcr.io -u gpsaggese -p ${{ secrets.GH_ACTION_ACCESS_TOKEN }} + + # Make everything accessible by any user to avoid permission errors. + - name: Cleanup + run: sudo chmod 777 -R . + + # Check out the code from GitHub so that we can run the action inside + # the Docker container. + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: true + # TODO(Samarth): Do we need to propagate this to other `repos/workflow` + # make it a default behavior? For certain tests to pass, we need entire + # commit history of the repo including sub-modules. + fetch-depth: 0 + token: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} + + # To see the modules in `helpers_root`, PYTHONPATH needs to include + # `helpers_root` in the same way we do in `setenv.sh`. + - name: Update PYTHONPATH + run: echo "PYTHONPATH=.:helpers_root" >> $GITHUB_ENV + + # Install packages that are required to run the job via GH. + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r .github/gh_requirements.txt + + # Pull the latest Docker image from the GHCR registry instead of ECR for + # cost saving purposes to run the regressions on. + - name: Pull image from GHCR + run: docker pull ghcr.io/${{ github.repository }}:dev + + # This step is used to trigger the fast test coverage generation using the invoke task. + - name: Run Fast test and generate report + id: run_fast + continue-on-error: true + env: + GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} + CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} + CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} + CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} + run: invoke run_coverage --suite fast + + - name: Upload Fast Test Coverage to Codecov + id: upload_fast + # Only upload if the previous fast test run step succeeded (i.r report generated). + # failed step don’t generate a coverage report, so there's nothing to upload. + if: steps.run_fast.outcome == 'success' + continue-on-error: true + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + # Specify the Codecov flag name associated with this test suite. + # Required to separate coverage reports by type (e.g., fast, slow, superslow) inside the Codecov UI. + flags: fast + name: fast-test-coverage + + - name: Run Slow test and generate report + id: run_slow + continue-on-error: true + env: + GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} + CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} + CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} + CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} + run: invoke run_coverage --suite slow + + - name: Upload Slow Test Coverage to Codecov + id: upload_slow + # Only upload if the previous slow test run step succeeded (i.e, if report generated). + if: steps.run_slow.outcome == 'success' + continue-on-error: true + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + flags: slow + name: slow-test-coverage + + - name: Run Superslow test and generate report + id: run_superslow + env: + GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} + CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} + CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} + CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} + run: | + # Determine the day of the week (1 = Monday, 7 = Sunday). + day_of_week=$(date +%u) + # Only run superslow tests on Mondays or if the workflow is manually triggered. + if [ "$day_of_week" = "1" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "Running superslow tests..." + invoke run_coverage --suite superslow + else + echo "Skipping superslow tests — today is not Monday and this is not a manual trigger" + exit 0 + fi + + - name: Upload Superslow Test Coverage to Codecov + #TODO(Shaunak): Consider removing it when we turn this workflow into a reusable one. + if: steps.run_superslow.outcome == 'success' + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + flags: superslow + name: superslow-test-coverage + + # Fail the job in CI if any of the fast/ slow run/ upload steps above failed. + - name: Fail if fast/slow test or upload failed + run: | + failed="" + if [ "${{ steps.run_fast.outcome }}" != "success" ]; then + echo "Fast test run failed" + failed="true" + fi + if [ "${{ steps.upload_fast.outcome }}" != "success" ]; then + echo "Fast test coverage upload failed" + failed="true" + fi + if [ "${{ steps.run_slow.outcome }}" != "success" ]; then + echo "Slow test run failed" + failed="true" + fi + if [ "${{ steps.upload_slow.outcome }}" != "success" ]; then + echo "Slow test coverage upload failed" + failed="true" + fi + if [ "$failed" = "true" ]; then + echo "At least one fast/slow test or upload step failed." + exit 1 + fi