diff --git a/.github/workflows/_build-app.yml b/.github/workflows/_build-app.yml index 871e28239a..28da5744c9 100644 --- a/.github/workflows/_build-app.yml +++ b/.github/workflows/_build-app.yml @@ -45,49 +45,19 @@ jobs: run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev - - name: Cache Requirements - id: cache-requirements - uses: actions/cache@v4 - env: - # Forks can't access the variable containing our actual image repository. We want to - # use a separate cache to make sure they don't interfere with reqs images being pushed. - cache-name: ${{ !github.event.pull_request.repo.fork && 'umbrella-requirements' || 'umbrella-requirements-fork' }} - with: - path: | - ./requirements.tar - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('libs/shared/**') }} - - - name: Cache App - id: cache-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-app - with: - path: | - ${{ inputs.output_directory }}/app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - - name: Load requirements from cache - if: ${{ steps.cache-requirements.outputs.cache-hit == 'true' }} - run: | - make load.requirements - - # This shouldn't happen; the _build-requirements.yml job should have run. - - name: Build/pull requirements - if: ${{ steps.cache-requirements.outputs.cache-hit != 'true' }} + - name: Build app run: | - echo "Warning: requirements image not in cache, building a new one" make build.requirements - make save.requirements + make ${{ inputs.make_target_prefix }}build.app - - name: Build app + - name: Push app + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' && inputs.make_target_prefix != 'shared.' }} run: | - make ${{ inputs.make_target_prefix }}build.app - make ${{ inputs.make_target_prefix }}save.app + make ${{ inputs.make_target_prefix }}push.app-temp build-test-app: name: Build Test App - runs-on: ubuntu-large + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -109,40 +79,13 @@ jobs: run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev - - name: Cache Test Requirements - id: cache-test-requirements - uses: actions/cache@v4 - env: - cache-name: umbrella-test-requirements - with: - path: | - ./test-requirements.tar - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('./uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('docker/Dockerfile.test-requirements') }}-${{ hashFiles('libs/shared/**') }} - - - name: Cache Test App - id: cache-test-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-test-app - with: - path: | - ${{ inputs.output_directory }}/test-app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - - name: Load test requirements from cache - if: ${{ steps.cache-test-requirements.outputs.cache-hit == 'true' }} - run: | - make load.test-requirements - - # This shouldn't happen; the _build-requirements.yml job should have run. - - name: Build/pull test requirements - if: ${{ steps.cache-test-requirements.outputs.cache-hit != 'true' }} + - name: Build Test App run: | - echo "Warning: test requirements image not in cache, building a new one" make build.test-requirements - make save.test-requirements + make ${{ inputs.make_target_prefix }}build.test-app - - name: Build Test App + - name: Push Test App + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' && inputs.make_target_prefix != 'shared.' }} run: | - make ${{ inputs.make_target_prefix }}build.test-app - make ${{ inputs.make_target_prefix }}save.test-app + make ${{ inputs.make_target_prefix }}push.test-app-temp + diff --git a/.github/workflows/_build-requirements.yml b/.github/workflows/_build-requirements.yml index 1d928da62e..0345478c63 100644 --- a/.github/workflows/_build-requirements.yml +++ b/.github/workflows/_build-requirements.yml @@ -16,39 +16,8 @@ jobs: with: fetch-depth: 2 - ###### - # Setting up caches for the base and test requirements images. - # - # If both caches hit, we don't need to do anything else. - # Otherwise, we have to auth with GCP and Docker Hub, build the missing - # images, and then push them. - ###### - - name: Cache Requirements - id: cache-requirements - uses: actions/cache@v4 - env: - # Forks can't access the variable containing our actual image repository. We want to - # use a separate cache to make sure they don't interfere with reqs images being pushed. - cache-name: ${{ !github.event.pull_request.repo.fork && 'umbrella-requirements' || 'umbrella-requirements-fork' }} - with: - path: | - ./requirements.tar - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('./uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('libs/shared/**') }} - - - name: Cache Test Requirements - id: cache-test-requirements - uses: actions/cache@v4 - env: - cache-name: umbrella-test-requirements - with: - path: | - ./test-requirements.tar - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('./uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('docker/Dockerfile.test-requirements') }}-${{ hashFiles('libs/shared/**') }} - - id: "auth" - if: | - (steps.cache-requirements.outputs.cache-hit != 'true' || steps.cache-test-requirements.outputs.cache-hit != 'true') && - !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} name: "Authenticate to Google Cloud" uses: "google-github-actions/auth@v2.1.2" with: @@ -57,36 +26,30 @@ jobs: service_account: ${{ secrets.CODECOV_GCP_WIDSA }} - name: Docker configuration - if: | - (steps.cache-requirements.outputs.cache-hit != 'true' || steps.cache-test-requirements.outputs.cache-hit != 'true') && - !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev ###### - # Building/pushing the base requirements image if not cached + # Building/pushing the base requirements image ###### - name: Build/pull requirements - if: ${{ steps.cache-requirements.outputs.cache-hit != 'true' }} run: | make build.requirements - make save.requirements - name: Push Requirements - if: ${{ steps.cache-requirements.outputs.cache-hit != 'true' && !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} run: | make push.requirements ###### - # Building/pushing the test requirements image if not cached + # Building/pushing the test requirements image ###### - name: Build/pull test requirements - if: ${{ steps.cache-test-requirements.outputs.cache-hit != 'true' }} run: | make build.test-requirements - make save.test-requirements - name: Push Test Requirements - if: ${{ steps.cache-test-requirements.outputs.cache-hit != 'true' && !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} run: | make push.test-requirements diff --git a/.github/workflows/_push-env.yml b/.github/workflows/_push-env.yml index f4795d4903..d0005b12a4 100644 --- a/.github/workflows/_push-env.yml +++ b/.github/workflows/_push-env.yml @@ -46,23 +46,13 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' + - name: Get Release SHA env: SHA: ${{ github.sha }} id: sha run: echo short_sha="${SHA:0:7}" >> $GITHUB_OUTPUT - - name: Cache App - id: cache-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-app - with: - path: | - ${{ inputs.output_directory }}/app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image - run: | - docker load --input ${{ inputs.output_directory }}/app.tar + - id: "auth" name: "Authenticate to Google Cloud" uses: "google-github-actions/auth@v2.1.2" @@ -75,10 +65,15 @@ jobs: run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev + + - name: Pull built image + run: | + make ${{ inputs.make_target_prefix }}pull.app-temp - name: Push ${{ inputs.environment }} run: | make ${{ inputs.make_target_prefix }}tag.${{ inputs.environment }} make ${{ inputs.make_target_prefix }}push.${{ inputs.environment }} + - name: Push latest if: inputs.environment == 'production' run: | @@ -106,18 +101,9 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' - - name: Cache App - id: cache-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-app - with: - path: | - ${{ inputs.output_directory }}/app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image + - name: Pull built image run: | - docker load --input ${{ inputs.output_directory }}/app.tar + make ${{ inputs.make_target_prefix }}pull.app-temp - name: Log in to Docker Hub uses: docker/login-action@v3.1.0 with: @@ -137,18 +123,9 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' - - name: Cache App - id: cache-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-app - with: - path: | - ${{ inputs.output_directory }}/app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image + - name: Pull built image run: | - docker load --input ${{ inputs.output_directory }}/app.tar + make ${{ inputs.make_target_prefix }}pull.app-temp - name: Log in to Docker Hub uses: docker/login-action@v3.1.0 with: diff --git a/.github/workflows/_run-tests.yml b/.github/workflows/_run-tests.yml index ae152898e5..4b84300a01 100644 --- a/.github/workflows/_run-tests.yml +++ b/.github/workflows/_run-tests.yml @@ -41,18 +41,26 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' - - name: Cache Test App - id: cache-test-app - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-test-app + + - id: "auth" + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v2.1.2" with: - path: | - ${{ inputs.output_directory }}/test-app.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image + token_format: "access_token" + workload_identity_provider: ${{ secrets.CODECOV_GCP_WIDP }} + service_account: ${{ secrets.CODECOV_GCP_WIDSA }} + + - name: Docker configuration + if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} + run: |- + echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev + + - name: Pull or build test image run: | - docker load --input ${{ inputs.output_directory }}/test-app.tar + echo "Pulling test image from registry" + make ${{ inputs.make_target_prefix }}pull.test-app-temp + - name: Install docker compose run: | sudo curl -SL https://github.com/docker/compose/releases/download/v2.20.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose diff --git a/.github/workflows/_self-hosted.yml b/.github/workflows/_self-hosted.yml index 07c639d915..1079a30e07 100644 --- a/.github/workflows/_self-hosted.yml +++ b/.github/workflows/_self-hosted.yml @@ -30,6 +30,7 @@ env: jobs: build-self-hosted: name: Build Self Hosted App + if: github.repository_owner == 'codecov' && !github.event.pull_request.head.repo.fork runs-on: ubuntu-latest steps: - name: Checkout @@ -51,52 +52,13 @@ jobs: run: |- echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev - - name: Cache Requirements - id: cache-requirements - uses: actions/cache@v4 - env: - # Forks can't access the variable containing our actual image repository. We want to - # use a separate cache to make sure they don't interfere with reqs images being pushed. - cache-name: ${{ !github.event.pull_request.repo.fork && 'umbrella-requirements' || 'umbrella-requirements-fork' }} - with: - path: | - ./requirements.tar - key: ${{ runner.os }}-${{ runner.arch }}-${{ env.cache-name }}-${{ hashFiles('uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('libs/shared/**') }} - - - name: Cache Self hosted - id: cache-self-hosted - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-self-hosted - with: - path: | - ${{ inputs.output_directory }}/self-hosted-runtime.tar - ${{ inputs.output_directory }}/self-hosted.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - - name: Load requirements from cache - if: ${{ steps.cache-requirements.outputs.cache-hit == 'true' }} - run: | - make load.requirements - - # This shouldn't happen; the _build-requirements.yml job should have run. - - name: Build/pull requirements - if: ${{ steps.cache-requirements.outputs.cache-hit != 'true' }} - run: | - echo "Warning: requirements image not in cache, building a new one" - make build.requirements - make save.requirements - - - name: Load built image - if: ${{ steps.cache-self-hosted.outputs.cache-hit == 'true' }} - run: | - make ${{ inputs.make_target_prefix }}load.self-hosted - - name: Build self hosted - if: ${{ steps.cache-self-hosted.outputs.cache-hit != 'true' }} run: | make ${{ inputs.make_target_prefix }}build.self-hosted - make ${{ inputs.make_target_prefix }}save.self-hosted + + - name: Push self hosted + run: | + make ${{ inputs.make_target_prefix }}push.self-hosted-temp self-hosted: name: Push Self Hosted Image @@ -110,19 +72,21 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' - - name: Cache Self hosted - id: cache-self-hosted - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-self-hosted + - id: "auth" + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v2.1.2" with: - path: | - ${{ inputs.output_directory }}/self-hosted-runtime.tar - ${{ inputs.output_directory }}/self-hosted.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image + token_format: "access_token" + workload_identity_provider: ${{ secrets.CODECOV_GCP_WIDP }} + service_account: ${{ secrets.CODECOV_GCP_WIDSA }} + + - name: Docker configuration + run: |- + echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev + + - name: Pull built images run: | - make ${{ inputs.make_target_prefix }}load.self-hosted + make ${{ inputs.make_target_prefix }}pull.self-hosted-temp - name: Log in to Docker Hub uses: docker/login-action@v3.1.0 with: @@ -144,19 +108,22 @@ jobs: with: fetch-depth: 0 submodules: 'recursive' - - name: Cache Self hosted - id: cache-self-hosted - uses: actions/cache@v4 - env: - cache-name: ${{ inputs.repo }}-self-hosted + - id: "auth" + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v2.1.2" with: - path: | - ${{ inputs.output_directory }}/self-hosted-runtime.tar - ${{ inputs.output_directory }}/self-hosted.tar - key: ${{ runner.os }}-${{ env.cache-name }}-${{ github.run_id }} - - name: Load built image + token_format: "access_token" + workload_identity_provider: ${{ secrets.CODECOV_GCP_WIDP }} + service_account: ${{ secrets.CODECOV_GCP_WIDSA }} + + - name: Docker configuration + run: |- + echo ${{steps.auth.outputs.access_token}} | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev + + - name: Pull built images run: | - make ${{ inputs.make_target_prefix }}load.self-hosted + make ${{ inputs.make_target_prefix }}pull.self-hosted-temp + - name: Log in to Docker Hub uses: docker/login-action@v3.1.0 with: diff --git a/docker/Makefile.docker b/docker/Makefile.docker index 02f78387ce..69e98fb5c3 100644 --- a/docker/Makefile.docker +++ b/docker/Makefile.docker @@ -35,19 +35,25 @@ export TEST_REQS_TAG := test-$(shell echo ${DOCKER_TEST_REQS_SHA} ${REQUIREMENTS # Build a requirements image. build.requirements: docker pull ${AR_REQS_REPO}:${REQUIREMENTS_TAG} || docker build \ - --network host \ - -f docker/Dockerfile.requirements . \ - -t ${AR_REQS_REPO}:${REQUIREMENTS_TAG} \ - -t ${CI_REQS_REPO}:${REQUIREMENTS_TAG} + --network host \ + -f docker/Dockerfile.requirements . \ + -t ${AR_REQS_REPO}:${REQUIREMENTS_TAG} \ + -t ${CI_REQS_REPO}:${REQUIREMENTS_TAG} # Build a test requirements image. Requires that the base reqs image exist. build.test-requirements: docker pull ${AR_REQS_REPO}:${TEST_REQS_TAG} || docker build \ - --network host \ - -f docker/Dockerfile.test-requirements . \ - -t ${AR_REQS_REPO}:${TEST_REQS_TAG} \ - -t ${CI_REQS_REPO}:${TEST_REQS_TAG} \ - --build-arg REQUIREMENTS_IMAGE=${AR_REQS_REPO}:${REQUIREMENTS_TAG} + --network host \ + -f docker/Dockerfile.test-requirements . \ + -t ${AR_REQS_REPO}:${TEST_REQS_TAG} \ + -t ${CI_REQS_REPO}:${TEST_REQS_TAG} \ + --build-arg REQUIREMENTS_IMAGE=${AR_REQS_REPO}:${REQUIREMENTS_TAG} + +# Build test requirements image and the test app +build.test-app: + $(MAKE) build.requirements + $(MAKE) build.test-requirements + $(MAKE) _build.test-app # Build an image for local development. _build.local: @@ -97,8 +103,10 @@ _build.test-app: --build-arg RELEASE_VERSION=${VERSION} \ --build-arg BUILD_ENV=cloud + # Build a self-hosted image. _build.self-hosted: + $(MAKE) build.requirements $(MAKE) _build.self-hosted-base $(MAKE) _build.self-hosted-runtime @@ -147,49 +155,35 @@ _tag.self-hosted-release: docker tag ${DOCKERHUB_REPO}:${VERSION} ${DOCKERHUB_REPO}:latest-stable docker tag ${DOCKERHUB_REPO}:${VERSION} ${DOCKERHUB_REPO}:latest-calver -# Load an exported requirements image (e.g. from GHA's cache) -load.requirements: - docker load --input requirements.tar - docker tag ${CI_REQS_REPO}:${REQUIREMENTS_TAG} ${AR_REQS_REPO}:${REQUIREMENTS_TAG} - -# Load an exported test requirements image (e.g. from GHA's cache) -load.test-requirements: - docker load --input test-requirements.tar - docker tag ${CI_REQS_REPO}:${TEST_REQS_TAG} ${AR_REQS_REPO}:${TEST_REQS_TAG} - -# Load an exported self-hosted image (e.g. from GHA's cache) -_load.self-hosted: - docker load --input ${APP_DIR}/self-hosted-runtime.tar - docker load --input ${APP_DIR}/self-hosted.tar - -# Export a production image. -_save.app: - docker save -o ${APP_DIR}/app.tar ${AR_REPO}:${VERSION} - -# Export a production image with test dependencies. -_save.test-app: - docker save -o ${APP_DIR}/test-app.tar ${AR_REPO}:test-${VERSION} - -# Export a requirements image. -save.requirements: - docker tag ${AR_REQS_REPO}:${REQUIREMENTS_TAG} ${CI_REQS_REPO}:${REQUIREMENTS_TAG} - docker save -o requirements.tar ${CI_REQS_REPO}:${REQUIREMENTS_TAG} - -# Export a test requirements image. -save.test-requirements: - docker tag ${AR_REQS_REPO}:${TEST_REQS_TAG} ${CI_REQS_REPO}:${TEST_REQS_TAG} - docker save -o test-requirements.tar ${CI_REQS_REPO}:${TEST_REQS_TAG} - -# Export a self-hosted image pair (base and runtime) -_save.self-hosted: - $(MAKE) _save.self-hosted-base - $(MAKE) _save.self-hosted-runtime - -_save.self-hosted-base: - docker save -o ${APP_DIR}/self-hosted.tar ${DOCKERHUB_REPO}:${VERSION}-no-dependencies - -_save.self-hosted-runtime: - docker save -o ${APP_DIR}/self-hosted-runtime.tar ${DOCKERHUB_REPO}:${VERSION} +# Push self-hosted images with temporary tags (for passing between CI jobs) +_push.self-hosted-temp: + docker tag ${DOCKERHUB_REPO}:${VERSION}-no-dependencies ${AR_REPO}:self-hosted-base-${VERSION} + docker tag ${DOCKERHUB_REPO}:${VERSION} ${AR_REPO}:self-hosted-runtime-${VERSION} + docker push ${AR_REPO}:self-hosted-base-${VERSION} + docker push ${AR_REPO}:self-hosted-runtime-${VERSION} + +# Pull self-hosted images from temporary tags +_pull.self-hosted-temp: + docker pull ${AR_REPO}:self-hosted-base-${VERSION} + docker pull ${AR_REPO}:self-hosted-runtime-${VERSION} + docker tag ${AR_REPO}:self-hosted-base-${VERSION} ${DOCKERHUB_REPO}:${VERSION}-no-dependencies + docker tag ${AR_REPO}:self-hosted-runtime-${VERSION} ${DOCKERHUB_REPO}:${VERSION} + +# Push production app with temporary tag (for passing between CI jobs) +_push.app-temp: + docker push ${AR_REPO}:${VERSION} + +# Pull production app from temporary tag +_pull.app-temp: + docker pull ${AR_REPO}:${VERSION} + +# Push test app with temporary tag (for passing between CI jobs) +_push.test-app-temp: + docker push ${AR_REPO}:test-${VERSION} + +# Pull test app from temporary tag or build +_pull.test-app-temp: + docker pull ${AR_REPO}:test-${VERSION} || $(MAKE) build.test-app _push.latest: docker push ${AR_REPO}:latest