From 2cf25cc67ad1d1729a917fb6d1f400f6c0ca4778 Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Thu, 16 Oct 2025 10:53:43 -0400 Subject: [PATCH 1/8] Refactor GitHub Actions workflows to streamline caching and build processes - Removed caching steps for requirements and test requirements from multiple workflows. - Updated build steps to directly invoke requirements and test requirements builds. - Simplified conditions for authentication and Docker configuration based on repository ownership. - Ensured that the self-hosted build job only runs for the appropriate repository owner. --- .github/workflows/_build-app.yml | 50 +---------------------- .github/workflows/_build-requirements.yml | 49 +++------------------- .github/workflows/_self-hosted.yml | 26 +----------- docker/Makefile.docker | 1 + 4 files changed, 10 insertions(+), 116 deletions(-) diff --git a/.github/workflows/_build-app.yml b/.github/workflows/_build-app.yml index 871e28239a..e65c458b23 100644 --- a/.github/workflows/_build-app.yml +++ b/.github/workflows/_build-app.yml @@ -45,18 +45,6 @@ jobs: run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev - - name: Cache Requirements - id: cache-requirements - uses: actions/cache@v4 - env: - # Forks can't access the variable containing our actual image repository. We want to - # use a separate cache to make sure they don't interfere with reqs images being pushed. - cache-name: ${{ !github.event.pull_request.repo.fork && 'umbrella-requirements' || 'umbrella-requirements-fork' }} - with: - path: | - ./requirements.tar - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('libs/shared/**') }} - - name: Cache App id: cache-app uses: actions/cache@v4 @@ -67,21 +55,9 @@ jobs: ${{ inputs.output_directory }}/app.tar key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load requirements from cache - if: ${{ steps.cache-requirements.outputs.cache-hit == 'true' }} - run: | - make load.requirements - - # This shouldn't happen; the _build-requirements.yml job should have run. - - name: Build/pull requirements - if: ${{ steps.cache-requirements.outputs.cache-hit != 'true' }} - run: | - echo "Warning: requirements image not in cache, building a new one" - make build.requirements - make save.requirements - - name: Build app run: | + make ${{ inputs.make_target_prefix }}build.requirements make ${{ inputs.make_target_prefix }}build.app make ${{ inputs.make_target_prefix }}save.app @@ -109,16 +85,6 @@ jobs: run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev - - name: Cache Test Requirements - id: cache-test-requirements - uses: actions/cache@v4 - env: - cache-name: umbrella-test-requirements - with: - path: | - ./test-requirements.tar - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('./uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('docker/Dockerfile.test-requirements') }}-${{ hashFiles('libs/shared/**') }} - - name: Cache Test App id: cache-test-app uses: actions/cache@v4 @@ -129,20 +95,8 @@ jobs: ${{ inputs.output_directory }}/test-app.tar key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load test requirements from cache - if: ${{ steps.cache-test-requirements.outputs.cache-hit == 'true' }} - run: | - make load.test-requirements - - # This shouldn't happen; the _build-requirements.yml job should have run. - - name: Build/pull test requirements - if: ${{ steps.cache-test-requirements.outputs.cache-hit != 'true' }} - run: | - echo "Warning: test requirements image not in cache, building a new one" - make build.test-requirements - make save.test-requirements - - name: Build Test App run: | + make ${{ inputs.make_target_prefix }}build.test-requirements make ${{ inputs.make_target_prefix }}build.test-app make ${{ inputs.make_target_prefix }}save.test-app diff --git a/.github/workflows/_build-requirements.yml b/.github/workflows/_build-requirements.yml index 1d928da62e..0345478c63 100644 --- a/.github/workflows/_build-requirements.yml +++ b/.github/workflows/_build-requirements.yml @@ -16,39 +16,8 @@ jobs: with: fetch-depth: 2 - ###### - # Setting up caches for the base and test requirements images. - # - # If both caches hit, we don't need to do anything else. - # Otherwise, we have to auth with GCP and Docker Hub, build the missing - # images, and then push them. - ###### - - name: Cache Requirements - id: cache-requirements - uses: actions/cache@v4 - env: - # Forks can't access the variable containing our actual image repository. We want to - # use a separate cache to make sure they don't interfere with reqs images being pushed. - cache-name: ${{ !github.event.pull_request.repo.fork && 'umbrella-requirements' || 'umbrella-requirements-fork' }} - with: - path: | - ./requirements.tar - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('./uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('libs/shared/**') }} - - - name: Cache Test Requirements - id: cache-test-requirements - uses: actions/cache@v4 - env: - cache-name: umbrella-test-requirements - with: - path: | - ./test-requirements.tar - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('./uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('docker/Dockerfile.test-requirements') }}-${{ hashFiles('libs/shared/**') }} - - id: "auth" - if: | - (steps.cache-requirements.outputs.cache-hit != 'true' || steps.cache-test-requirements.outputs.cache-hit != 'true') && - !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} name: "Authenticate to Google Cloud" uses: "google-github-actions/auth@v2.1.2" with: @@ -57,36 +26,30 @@ jobs: service_account: ${{ secrets.CODECOV_GCP_WIDSA }} - name: Docker configuration - if: | - (steps.cache-requirements.outputs.cache-hit != 'true' || steps.cache-test-requirements.outputs.cache-hit != 'true') && - !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev ###### - # Building/pushing the base requirements image if not cached + # Building/pushing the base requirements image ###### - name: Build/pull requirements - if: ${{ steps.cache-requirements.outputs.cache-hit != 'true' }} run: | make build.requirements - make save.requirements - name: Push Requirements - if: ${{ steps.cache-requirements.outputs.cache-hit != 'true' && !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} run: | make push.requirements ###### - # Building/pushing the test requirements image if not cached + # Building/pushing the test requirements image ###### - name: Build/pull test requirements - if: ${{ steps.cache-test-requirements.outputs.cache-hit != 'true' }} run: | make build.test-requirements - make save.test-requirements - name: Push Test Requirements - if: ${{ steps.cache-test-requirements.outputs.cache-hit != 'true' && !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} run: | make push.test-requirements diff --git a/.github/workflows/_self-hosted.yml b/.github/workflows/_self-hosted.yml index 07c639d915..ba3909bb8a 100644 --- a/.github/workflows/_self-hosted.yml +++ b/.github/workflows/_self-hosted.yml @@ -30,6 +30,7 @@ env: jobs: build-self-hosted: name: Build Self Hosted App + if: github.repository_owner == 'codecov' && !github.event.pull_request.head.repo.fork runs-on: ubuntu-latest steps: - name: Checkout @@ -51,18 +52,6 @@ jobs: run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev - - name: Cache Requirements - id: cache-requirements - uses: actions/cache@v4 - env: - # Forks can't access the variable containing our actual image repository. We want to - # use a separate cache to make sure they don't interfere with reqs images being pushed. - cache-name: ${{ !github.event.pull_request.repo.fork && 'umbrella-requirements' || 'umbrella-requirements-fork' }} - with: - path: | - ./requirements.tar - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('libs/shared/**') }} - - name: Cache Self hosted id: cache-self-hosted uses: actions/cache@v4 @@ -74,19 +63,6 @@ jobs: ${{ inputs.output_directory }}/self-hosted.tar key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load requirements from cache - if: ${{ steps.cache-requirements.outputs.cache-hit == 'true' }} - run: | - make load.requirements - - # This shouldn't happen; the _build-requirements.yml job should have run. - - name: Build/pull requirements - if: ${{ steps.cache-requirements.outputs.cache-hit != 'true' }} - run: | - echo "Warning: requirements image not in cache, building a new one" - make build.requirements - make save.requirements - - name: Load built image if: ${{ steps.cache-self-hosted.outputs.cache-hit == 'true' }} run: | diff --git a/docker/Makefile.docker b/docker/Makefile.docker index 02f78387ce..fb068da7d4 100644 --- a/docker/Makefile.docker +++ b/docker/Makefile.docker @@ -99,6 +99,7 @@ _build.test-app: # Build a self-hosted image. _build.self-hosted: + $(MAKE) build.requirements $(MAKE) _build.self-hosted-base $(MAKE) _build.self-hosted-runtime From 8816ab02464604b356e25c93827414714e0f50fc Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Thu, 16 Oct 2025 11:13:01 -0400 Subject: [PATCH 2/8] Refactor GitHub Actions workflows to enhance image handling and build processes - Removed caching steps for app and test app images across multiple workflows. - Introduced new steps to push and pull images with temporary tags for better CI job management. - Updated Docker commands to streamline image loading and authentication processes. - Consolidated image build steps in the Makefile for improved clarity and efficiency. --- .github/workflows/_build-app.yml | 32 +++++--------- .github/workflows/_push-env.yml | 39 +++------------- .github/workflows/_run-tests.yml | 28 +++++++----- .github/workflows/_self-hosted.yml | 71 +++++++++++++----------------- docker/Makefile.docker | 57 +++++++++++++++++------- 5 files changed, 105 insertions(+), 122 deletions(-) diff --git a/.github/workflows/_build-app.yml b/.github/workflows/_build-app.yml index e65c458b23..8801e7e5a1 100644 --- a/.github/workflows/_build-app.yml +++ b/.github/workflows/_build-app.yml @@ -45,21 +45,15 @@ jobs: run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev - - name: Cache App - id: cache-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-app - with: - path: | - ${{ inputs.output_directory }}/app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Build app run: | make ${{ inputs.make_target_prefix }}build.requirements make ${{ inputs.make_target_prefix }}build.app - make ${{ inputs.make_target_prefix }}save.app + + - name: Push app + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + run: | + make ${{ inputs.make_target_prefix }}push.app-temp build-test-app: name: Build Test App @@ -85,18 +79,12 @@ jobs: run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev - - name: Cache Test App - id: cache-test-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-test-app - with: - path: | - ${{ inputs.output_directory }}/test-app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Build Test App run: | make ${{ inputs.make_target_prefix }}build.test-requirements make ${{ inputs.make_target_prefix }}build.test-app - make ${{ inputs.make_target_prefix }}save.test-app + + - name: Push Test App + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + run: | + make ${{ inputs.make_target_prefix }}push.test-app-temp diff --git a/.github/workflows/_push-env.yml b/.github/workflows/_push-env.yml index f4795d4903..4ba2c1bc01 100644 --- a/.github/workflows/_push-env.yml +++ b/.github/workflows/_push-env.yml @@ -51,18 +51,9 @@ jobs: SHA: ${{ github.sha }} id: sha run: echo short_sha="${SHA:0:7}" >> $GITHUB_OUTPUT - - name: Cache App - id: cache-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-app - with: - path: | - ${{ inputs.output_directory }}/app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image + - name: Pull built image run: | - docker load --input ${{ inputs.output_directory }}/app.tar + make ${{ inputs.make_target_prefix }}pull.app-temp - id: "auth" name: "Authenticate to Google Cloud" uses: "google-github-actions/auth@v2.1.2" @@ -106,18 +97,9 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' - - name: Cache App - id: cache-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-app - with: - path: | - ${{ inputs.output_directory }}/app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image + - name: Pull built image run: | - docker load --input ${{ inputs.output_directory }}/app.tar + make ${{ inputs.make_target_prefix }}pull.app-temp - name: Log in to Docker Hub uses: docker/login-action@v3.1.0 with: @@ -137,18 +119,9 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' - - name: Cache App - id: cache-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-app - with: - path: | - ${{ inputs.output_directory }}/app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image + - name: Pull built image run: | - docker load --input ${{ inputs.output_directory }}/app.tar + make ${{ inputs.make_target_prefix }}pull.app-temp - name: Log in to Docker Hub uses: docker/login-action@v3.1.0 with: diff --git a/.github/workflows/_run-tests.yml b/.github/workflows/_run-tests.yml index ae152898e5..4b84300a01 100644 --- a/.github/workflows/_run-tests.yml +++ b/.github/workflows/_run-tests.yml @@ -41,18 +41,26 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' - - name: Cache Test App - id: cache-test-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-test-app + + - id: "auth" + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v2.1.2" with: - path: | - ${{ inputs.output_directory }}/test-app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image + token_format: "access_token" + workload_identity_provider: ${{ secrets.CODECOV_GCP_WIDP }} + service_account: ${{ secrets.CODECOV_GCP_WIDSA }} + + - name: Docker configuration + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + run: |- + echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev + + - name: Pull or build test image run: | - docker load --input ${{ inputs.output_directory }}/test-app.tar + echo "Pulling test image from registry" + make ${{ inputs.make_target_prefix }}pull.test-app-temp + - name: Install docker compose run: | sudo curl -SL https://github.com/docker/compose/releases/download/v2.20.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose diff --git a/.github/workflows/_self-hosted.yml b/.github/workflows/_self-hosted.yml index ba3909bb8a..1079a30e07 100644 --- a/.github/workflows/_self-hosted.yml +++ b/.github/workflows/_self-hosted.yml @@ -52,27 +52,13 @@ jobs: run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev - - name: Cache Self hosted - id: cache-self-hosted - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-self-hosted - with: - path: | - ${{ inputs.output_directory }}/self-hosted-runtime.tar - ${{ inputs.output_directory }}/self-hosted.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - - name: Load built image - if: ${{ steps.cache-self-hosted.outputs.cache-hit == 'true' }} - run: | - make ${{ inputs.make_target_prefix }}load.self-hosted - - name: Build self hosted - if: ${{ steps.cache-self-hosted.outputs.cache-hit != 'true' }} run: | make ${{ inputs.make_target_prefix }}build.self-hosted - make ${{ inputs.make_target_prefix }}save.self-hosted + + - name: Push self hosted + run: | + make ${{ inputs.make_target_prefix }}push.self-hosted-temp self-hosted: name: Push Self Hosted Image @@ -86,19 +72,21 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' - - name: Cache Self hosted - id: cache-self-hosted - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-self-hosted + - id: "auth" + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v2.1.2" with: - path: | - ${{ inputs.output_directory }}/self-hosted-runtime.tar - ${{ inputs.output_directory }}/self-hosted.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image + token_format: "access_token" + workload_identity_provider: ${{ secrets.CODECOV_GCP_WIDP }} + service_account: ${{ secrets.CODECOV_GCP_WIDSA }} + + - name: Docker configuration + run: |- + echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev + + - name: Pull built images run: | - make ${{ inputs.make_target_prefix }}load.self-hosted + make ${{ inputs.make_target_prefix }}pull.self-hosted-temp - name: Log in to Docker Hub uses: docker/login-action@v3.1.0 with: @@ -120,19 +108,22 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' - - name: Cache Self hosted - id: cache-self-hosted - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-self-hosted + - id: "auth" + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v2.1.2" with: - path: | - ${{ inputs.output_directory }}/self-hosted-runtime.tar - ${{ inputs.output_directory }}/self-hosted.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image + token_format: "access_token" + workload_identity_provider: ${{ secrets.CODECOV_GCP_WIDP }} + service_account: ${{ secrets.CODECOV_GCP_WIDSA }} + + - name: Docker configuration + run: |- + echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev + + - name: Pull built images run: | - make ${{ inputs.make_target_prefix }}load.self-hosted + make ${{ inputs.make_target_prefix }}pull.self-hosted-temp + - name: Log in to Docker Hub uses: docker/login-action@v3.1.0 with: diff --git a/docker/Makefile.docker b/docker/Makefile.docker index fb068da7d4..c8e3e8a04f 100644 --- a/docker/Makefile.docker +++ b/docker/Makefile.docker @@ -49,6 +49,12 @@ build.test-requirements: -t ${CI_REQS_REPO}:${TEST_REQS_TAG} \ --build-arg REQUIREMENTS_IMAGE=${AR_REQS_REPO}:${REQUIREMENTS_TAG} +# Build test requirements image and the test app +build.test-app: + $(MAKE) build.requirements + $(MAKE) build.test-requirements + $(MAKE) _build.test-app + # Build an image for local development. _build.local: docker build -f docker/Dockerfile . \ @@ -97,6 +103,7 @@ _build.test-app: --build-arg RELEASE_VERSION=${VERSION} \ --build-arg BUILD_ENV=cloud + # Build a self-hosted image. _build.self-hosted: $(MAKE) build.requirements @@ -158,18 +165,39 @@ load.test-requirements: docker load --input test-requirements.tar docker tag ${CI_REQS_REPO}:${TEST_REQS_TAG} ${AR_REQS_REPO}:${TEST_REQS_TAG} -# Load an exported self-hosted image (e.g. from GHA's cache) -_load.self-hosted: - docker load --input ${APP_DIR}/self-hosted-runtime.tar - docker load --input ${APP_DIR}/self-hosted.tar - -# Export a production image. -_save.app: - docker save -o ${APP_DIR}/app.tar ${AR_REPO}:${VERSION} - -# Export a production image with test dependencies. -_save.test-app: - docker save -o ${APP_DIR}/test-app.tar ${AR_REPO}:test-${VERSION} +# Push self-hosted images with temporary tags (for passing between CI jobs) +_push.self-hosted-temp: + docker tag ${DOCKERHUB_REPO}:${VERSION}-no-dependencies ${AR_REPO}:self-hosted-base-${GITHUB_RUN_ID} + docker tag ${DOCKERHUB_REPO}:${VERSION} ${AR_REPO}:self-hosted-runtime-${GITHUB_RUN_ID} + docker push ${AR_REPO}:self-hosted-base-${GITHUB_RUN_ID} + docker push ${AR_REPO}:self-hosted-runtime-${GITHUB_RUN_ID} + +# Pull self-hosted images from temporary tags +_pull.self-hosted-temp: + docker pull ${AR_REPO}:self-hosted-base-${GITHUB_RUN_ID} + docker pull ${AR_REPO}:self-hosted-runtime-${GITHUB_RUN_ID} + docker tag ${AR_REPO}:self-hosted-base-${GITHUB_RUN_ID} ${DOCKERHUB_REPO}:${VERSION}-no-dependencies + docker tag ${AR_REPO}:self-hosted-runtime-${GITHUB_RUN_ID} ${DOCKERHUB_REPO}:${VERSION} + +# Push production app with temporary tag (for passing between CI jobs) +_push.app-temp: + docker tag ${AR_REPO}:${VERSION} ${AR_REPO}:build-${GITHUB_RUN_ID} + docker push ${AR_REPO}:build-${GITHUB_RUN_ID} + +# Pull production app from temporary tag +_pull.app-temp: + docker pull ${AR_REPO}:build-${GITHUB_RUN_ID} + docker tag ${AR_REPO}:build-${GITHUB_RUN_ID} ${AR_REPO}:${VERSION} + +# Push test app with temporary tag (for passing between CI jobs) +_push.test-app-temp: + docker tag ${AR_REPO}:test-${VERSION} ${AR_REPO}:test-build-${GITHUB_RUN_ID} + docker push ${AR_REPO}:test-build-${GITHUB_RUN_ID} + +# Pull test app from temporary tag or build +_pull.test-app-temp: + docker pull ${AR_REPO}:test-build-${GITHUB_RUN_ID} || $(MAKE) build.test-app + docker tag ${AR_REPO}:test-build-${GITHUB_RUN_ID} ${AR_REPO}:test-${VERSION} # Export a requirements image. save.requirements: @@ -181,11 +209,6 @@ save.test-requirements: docker tag ${AR_REQS_REPO}:${TEST_REQS_TAG} ${CI_REQS_REPO}:${TEST_REQS_TAG} docker save -o test-requirements.tar ${CI_REQS_REPO}:${TEST_REQS_TAG} -# Export a self-hosted image pair (base and runtime) -_save.self-hosted: - $(MAKE) _save.self-hosted-base - $(MAKE) _save.self-hosted-runtime - _save.self-hosted-base: docker save -o ${APP_DIR}/self-hosted.tar ${DOCKERHUB_REPO}:${VERSION}-no-dependencies From 1d9974ca0bba0127c29b4635657b0ad08cf037ab Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Thu, 16 Oct 2025 12:17:19 -0400 Subject: [PATCH 3/8] Refactor Makefile and GitHub Actions workflows for improved build processes - Simplified build commands in the GitHub Actions workflow by removing the use of input variables for make targets. - Enhanced the Makefile by streamlining the build requirements and test requirements image creation steps. - Updated Docker commands to ensure consistent tagging and pulling of images, improving clarity and efficiency in CI processes. --- .github/workflows/_build-app.yml | 4 +- docker/Makefile.docker | 72 ++++++++++---------------------- 2 files changed, 23 insertions(+), 53 deletions(-) diff --git a/.github/workflows/_build-app.yml b/.github/workflows/_build-app.yml index 8801e7e5a1..7925368f1d 100644 --- a/.github/workflows/_build-app.yml +++ b/.github/workflows/_build-app.yml @@ -47,7 +47,7 @@ jobs: - name: Build app run: | - make ${{ inputs.make_target_prefix }}build.requirements + make build.requirements make ${{ inputs.make_target_prefix }}build.app - name: Push app @@ -81,7 +81,7 @@ jobs: - name: Build Test App run: | - make ${{ inputs.make_target_prefix }}build.test-requirements + make build.test-requirements make ${{ inputs.make_target_prefix }}build.test-app - name: Push Test App diff --git a/docker/Makefile.docker b/docker/Makefile.docker index c8e3e8a04f..69e98fb5c3 100644 --- a/docker/Makefile.docker +++ b/docker/Makefile.docker @@ -35,19 +35,19 @@ export TEST_REQS_TAG := test-$(shell echo ${DOCKER_TEST_REQS_SHA} ${REQUIREMENTS # Build a requirements image. build.requirements: docker pull ${AR_REQS_REPO}:${REQUIREMENTS_TAG} || docker build \ - --network host \ - -f docker/Dockerfile.requirements . \ - -t ${AR_REQS_REPO}:${REQUIREMENTS_TAG} \ - -t ${CI_REQS_REPO}:${REQUIREMENTS_TAG} + --network host \ + -f docker/Dockerfile.requirements . \ + -t ${AR_REQS_REPO}:${REQUIREMENTS_TAG} \ + -t ${CI_REQS_REPO}:${REQUIREMENTS_TAG} # Build a test requirements image. Requires that the base reqs image exist. build.test-requirements: docker pull ${AR_REQS_REPO}:${TEST_REQS_TAG} || docker build \ - --network host \ - -f docker/Dockerfile.test-requirements . \ - -t ${AR_REQS_REPO}:${TEST_REQS_TAG} \ - -t ${CI_REQS_REPO}:${TEST_REQS_TAG} \ - --build-arg REQUIREMENTS_IMAGE=${AR_REQS_REPO}:${REQUIREMENTS_TAG} + --network host \ + -f docker/Dockerfile.test-requirements . \ + -t ${AR_REQS_REPO}:${TEST_REQS_TAG} \ + -t ${CI_REQS_REPO}:${TEST_REQS_TAG} \ + --build-arg REQUIREMENTS_IMAGE=${AR_REQS_REPO}:${REQUIREMENTS_TAG} # Build test requirements image and the test app build.test-app: @@ -155,65 +155,35 @@ _tag.self-hosted-release: docker tag ${DOCKERHUB_REPO}:${VERSION} ${DOCKERHUB_REPO}:latest-stable docker tag ${DOCKERHUB_REPO}:${VERSION} ${DOCKERHUB_REPO}:latest-calver -# Load an exported requirements image (e.g. from GHA's cache) -load.requirements: - docker load --input requirements.tar - docker tag ${CI_REQS_REPO}:${REQUIREMENTS_TAG} ${AR_REQS_REPO}:${REQUIREMENTS_TAG} - -# Load an exported test requirements image (e.g. from GHA's cache) -load.test-requirements: - docker load --input test-requirements.tar - docker tag ${CI_REQS_REPO}:${TEST_REQS_TAG} ${AR_REQS_REPO}:${TEST_REQS_TAG} - # Push self-hosted images with temporary tags (for passing between CI jobs) _push.self-hosted-temp: - docker tag ${DOCKERHUB_REPO}:${VERSION}-no-dependencies ${AR_REPO}:self-hosted-base-${GITHUB_RUN_ID} - docker tag ${DOCKERHUB_REPO}:${VERSION} ${AR_REPO}:self-hosted-runtime-${GITHUB_RUN_ID} - docker push ${AR_REPO}:self-hosted-base-${GITHUB_RUN_ID} - docker push ${AR_REPO}:self-hosted-runtime-${GITHUB_RUN_ID} + docker tag ${DOCKERHUB_REPO}:${VERSION}-no-dependencies ${AR_REPO}:self-hosted-base-${VERSION} + docker tag ${DOCKERHUB_REPO}:${VERSION} ${AR_REPO}:self-hosted-runtime-${VERSION} + docker push ${AR_REPO}:self-hosted-base-${VERSION} + docker push ${AR_REPO}:self-hosted-runtime-${VERSION} # Pull self-hosted images from temporary tags _pull.self-hosted-temp: - docker pull ${AR_REPO}:self-hosted-base-${GITHUB_RUN_ID} - docker pull ${AR_REPO}:self-hosted-runtime-${GITHUB_RUN_ID} - docker tag ${AR_REPO}:self-hosted-base-${GITHUB_RUN_ID} ${DOCKERHUB_REPO}:${VERSION}-no-dependencies - docker tag ${AR_REPO}:self-hosted-runtime-${GITHUB_RUN_ID} ${DOCKERHUB_REPO}:${VERSION} + docker pull ${AR_REPO}:self-hosted-base-${VERSION} + docker pull ${AR_REPO}:self-hosted-runtime-${VERSION} + docker tag ${AR_REPO}:self-hosted-base-${VERSION} ${DOCKERHUB_REPO}:${VERSION}-no-dependencies + docker tag ${AR_REPO}:self-hosted-runtime-${VERSION} ${DOCKERHUB_REPO}:${VERSION} # Push production app with temporary tag (for passing between CI jobs) _push.app-temp: - docker tag ${AR_REPO}:${VERSION} ${AR_REPO}:build-${GITHUB_RUN_ID} - docker push ${AR_REPO}:build-${GITHUB_RUN_ID} + docker push ${AR_REPO}:${VERSION} # Pull production app from temporary tag _pull.app-temp: - docker pull ${AR_REPO}:build-${GITHUB_RUN_ID} - docker tag ${AR_REPO}:build-${GITHUB_RUN_ID} ${AR_REPO}:${VERSION} + docker pull ${AR_REPO}:${VERSION} # Push test app with temporary tag (for passing between CI jobs) _push.test-app-temp: - docker tag ${AR_REPO}:test-${VERSION} ${AR_REPO}:test-build-${GITHUB_RUN_ID} - docker push ${AR_REPO}:test-build-${GITHUB_RUN_ID} + docker push ${AR_REPO}:test-${VERSION} # Pull test app from temporary tag or build _pull.test-app-temp: - docker pull ${AR_REPO}:test-build-${GITHUB_RUN_ID} || $(MAKE) build.test-app - docker tag ${AR_REPO}:test-build-${GITHUB_RUN_ID} ${AR_REPO}:test-${VERSION} - -# Export a requirements image. -save.requirements: - docker tag ${AR_REQS_REPO}:${REQUIREMENTS_TAG} ${CI_REQS_REPO}:${REQUIREMENTS_TAG} - docker save -o requirements.tar ${CI_REQS_REPO}:${REQUIREMENTS_TAG} - -# Export a test requirements image. -save.test-requirements: - docker tag ${AR_REQS_REPO}:${TEST_REQS_TAG} ${CI_REQS_REPO}:${TEST_REQS_TAG} - docker save -o test-requirements.tar ${CI_REQS_REPO}:${TEST_REQS_TAG} - -_save.self-hosted-base: - docker save -o ${APP_DIR}/self-hosted.tar ${DOCKERHUB_REPO}:${VERSION}-no-dependencies - -_save.self-hosted-runtime: - docker save -o ${APP_DIR}/self-hosted-runtime.tar ${DOCKERHUB_REPO}:${VERSION} + docker pull ${AR_REPO}:test-${VERSION} || $(MAKE) build.test-app _push.latest: docker push ${AR_REPO}:latest From 5ba1bbde3ffcb54fd5ec1ae444885eeadbc5f03a Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Thu, 16 Oct 2025 15:40:50 -0400 Subject: [PATCH 4/8] Enhance GitHub Actions workflows for conditional image pushing and authentication - Updated conditions for pushing app and test app images to exclude shared targets. - Added a new step for authenticating to Google Cloud using workload identity and service account secrets. - Introduced a step to retrieve the short SHA for better traceability in the workflow. --- .github/workflows/_build-app.yml | 4 ++-- .github/workflows/_push-env.yml | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_build-app.yml b/.github/workflows/_build-app.yml index 7925368f1d..642b28626e 100644 --- a/.github/workflows/_build-app.yml +++ b/.github/workflows/_build-app.yml @@ -51,7 +51,7 @@ jobs: make ${{ inputs.make_target_prefix }}build.app - name: Push app - if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' && inputs.make_target_prefix != 'shared.' }} run: | make ${{ inputs.make_target_prefix }}push.app-temp @@ -85,6 +85,6 @@ jobs: make ${{ inputs.make_target_prefix }}build.test-app - name: Push Test App - if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' && inputs.make_target_prefix != 'shared.' }} run: | make ${{ inputs.make_target_prefix }}push.test-app-temp diff --git a/.github/workflows/_push-env.yml b/.github/workflows/_push-env.yml index 4ba2c1bc01..582a181fbf 100644 --- a/.github/workflows/_push-env.yml +++ b/.github/workflows/_push-env.yml @@ -46,11 +46,21 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' + - name: Get Release SHA env: SHA: ${{ github.sha }} id: sha run: echo short_sha="${SHA:0:7}" >> $GITHUB_OUTPUT + + - id: "auth" + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v2.1.2" + with: + token_format: "access_token" + workload_identity_provider: ${{ secrets.CODECOV_GCP_WIDP }} + service_account: ${{ secrets.CODECOV_GCP_WIDSA }} + - name: Pull built image run: | make ${{ inputs.make_target_prefix }}pull.app-temp From 0119369fcd1db1b54e67b08d5ecb23a0a3fc2a85 Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Thu, 16 Oct 2025 16:02:28 -0400 Subject: [PATCH 5/8] Refactor GitHub Actions workflow to improve image handling and organization - Adjusted the order of steps for pulling built images and Docker configuration for better clarity. - Removed redundant authentication step to streamline the workflow. - Ensured consistent formatting and spacing for improved readability. --- .github/workflows/_push-env.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/_push-env.yml b/.github/workflows/_push-env.yml index 582a181fbf..d0005b12a4 100644 --- a/.github/workflows/_push-env.yml +++ b/.github/workflows/_push-env.yml @@ -46,7 +46,7 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' - + - name: Get Release SHA env: SHA: ${{ github.sha }} @@ -61,25 +61,19 @@ jobs: workload_identity_provider: ${{ secrets.CODECOV_GCP_WIDP }} service_account: ${{ secrets.CODECOV_GCP_WIDSA }} - - name: Pull built image - run: | - make ${{ inputs.make_target_prefix }}pull.app-temp - - id: "auth" - name: "Authenticate to Google Cloud" - uses: "google-github-actions/auth@v2.1.2" - with: - token_format: "access_token" - workload_identity_provider: ${{ secrets.CODECOV_GCP_WIDP }} - service_account: ${{ secrets.CODECOV_GCP_WIDSA }} - - name: Docker configuration run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev + + - name: Pull built image + run: | + make ${{ inputs.make_target_prefix }}pull.app-temp - name: Push ${{ inputs.environment }} run: | make ${{ inputs.make_target_prefix }}tag.${{ inputs.environment }} make ${{ inputs.make_target_prefix }}push.${{ inputs.environment }} + - name: Push latest if: inputs.environment == 'production' run: | From 76048ea9bcd1cf5266dba8f9732ee0f1757d5201 Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Thu, 16 Oct 2025 16:09:01 -0400 Subject: [PATCH 6/8] Update GitHub Actions workflow to use standard Ubuntu runner - Changed the runner from 'ubuntu-large' to 'ubuntu' for the build-test-app job, optimizing resource usage. --- .github/workflows/_build-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_build-app.yml b/.github/workflows/_build-app.yml index 642b28626e..2ae22dcbb9 100644 --- a/.github/workflows/_build-app.yml +++ b/.github/workflows/_build-app.yml @@ -57,7 +57,7 @@ jobs: build-test-app: name: Build Test App - runs-on: ubuntu-large + runs-on: ubuntu steps: - name: Checkout uses: actions/checkout@v4 From ec343399dcbe14d36842631670e4d34619795823 Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Fri, 17 Oct 2025 13:49:23 -0400 Subject: [PATCH 7/8] Test change --- .github/workflows/_build-app.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/_build-app.yml b/.github/workflows/_build-app.yml index 2ae22dcbb9..400556b58a 100644 --- a/.github/workflows/_build-app.yml +++ b/.github/workflows/_build-app.yml @@ -88,3 +88,5 @@ jobs: if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' && inputs.make_target_prefix != 'shared.' }} run: | make ${{ inputs.make_target_prefix }}push.test-app-temp + + \ No newline at end of file From 43707ae2d5dfe5cae71492568e9ae1ec090da2cf Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Fri, 17 Oct 2025 13:57:35 -0400 Subject: [PATCH 8/8] Update GitHub Actions workflow to use the latest Ubuntu runner - Changed the runner from 'ubuntu' to 'ubuntu-latest' for the build-test-app job to ensure access to the latest features and updates. --- .github/workflows/_build-app.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/_build-app.yml b/.github/workflows/_build-app.yml index 400556b58a..28da5744c9 100644 --- a/.github/workflows/_build-app.yml +++ b/.github/workflows/_build-app.yml @@ -57,7 +57,7 @@ jobs: build-test-app: name: Build Test App - runs-on: ubuntu + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -89,4 +89,3 @@ jobs: run: | make ${{ inputs.make_target_prefix }}push.test-app-temp - \ No newline at end of file