From bce86def1ac7bd7d1fc53e5e3f9f7c6cd2261e7d Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Tue, 18 Mar 2025 13:02:57 +0100 Subject: [PATCH 1/6] Add kind integration test Signed-off-by: Enrico Minack --- .github/workflows/ci.yml | 4 ++++ .github/workflows/integration.yml | 32 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee6b523..2e73719 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,9 @@ jobs: build: if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id uses: ./.github/workflows/build.yml + integration: + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id + uses: ./.github/workflows/integration.yml # Virtual job that can be configured as a required check before a PR can be merged. all-required-checks-done: name: All required checks done @@ -40,6 +43,7 @@ jobs: - test - codeql - build + - integration runs-on: ubuntu-latest steps: - run: echo "All required checks done" diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..403bb8a --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,32 @@ +name: Integration + +on: + workflow_call: + +permissions: + contents: read + +jobs: + kind: + name: Kind + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Go + id: setup-go + uses: ./.github/actions/setup-go-cache + with: + cache-prefix: go-integration-kind + + - name: Starting Armada cluster + run: make kind-all + + - name: Create test queue + run: ./bin/app/armadactl create queue test + + - name: Test queue exists + run: ./bin/app/armadactl get queue test + + - name: Submit job + run: ./bin/app/armadactl submit dev/quickstart/example-job.yaml From 4bbced9f0faf40d1942b3b1f24c5d2ef87df7cff Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Tue, 18 Mar 2025 14:15:24 +0100 Subject: [PATCH 2/6] Retry creating and getting queue, submitting job Signed-off-by: Enrico Minack --- .github/workflows/integration.yml | 71 +++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 403bb8a..995e7f8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -22,11 +22,74 @@ jobs: - name: Starting Armada cluster run: make kind-all - - name: Create test queue - run: ./bin/app/armadactl create queue test + - name: Create queue + run: | + # Create queue + for second in {1..60} + do + if ./bin/app/armadactl create queue example 2> armadactl-${second}.log + then + if [[ second -eq 1 ]] + then + echo "Successfully created queue 'example'" + else + echo "Successfully created queue 'example' after ${second}s" + echo "Previously failed due to:" + cat armadactl-$((second-1)).log + fi + exit + fi + sleep 1 + done + + echo "Failed to create queue 'example' after ${second}s" + cat armadactl-${second}.log >&2 + exit 1 - name: Test queue exists - run: ./bin/app/armadactl get queue test + run: | + # Test queue exists + for second in {1..60} + do + if ./bin/app/armadactl get queue example 2> armadactl-${second}.log + then + if [[ second -eq 1 ]] + then + echo "Successfully inspected queue 'example'" + else + echo "Successfully inspected queue 'example' after ${second}s" + echo "Previously failed due to:" + cat armadactl-$((second-1)).log + fi + exit + fi + sleep 1 + done + + echo "Failed to inspected queue 'example' after ${second}s" + cat armadactl-${second}.log >&2 + exit 1 - name: Submit job - run: ./bin/app/armadactl submit dev/quickstart/example-job.yaml + run: | + # Submit job + for second in {1..60} + do + if ./bin/app/armadactl submit dev/quickstart/example-job.yaml 2> armadactl-${second}.log + then + if [[ attempt -eq 1 ]] + then + echo "Successfully submitted job" + else + echo "Successfully submitted job after ${second}s" + echo "Previously failed due to:" + cat armadactl-$((second-1)).log + fi + exit + fi + sleep 1 + done + + echo "Failed to submitted job after ${second}s" + cat armadactl-${second}.log >&2 + exit 1 From 7c7abca08c1539d45920007a8cdc94f2c7ba42c8 Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Wed, 19 Mar 2025 07:33:49 +0100 Subject: [PATCH 3/6] Rename integration to e2e Signed-off-by: Enrico Minack --- .github/workflows/ci.yml | 6 +++--- .github/workflows/{integration.yml => e2e.yml} | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename .github/workflows/{integration.yml => e2e.yml} (99%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e73719..67d1656 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,9 +31,9 @@ jobs: build: if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id uses: ./.github/workflows/build.yml - integration: + e2e: if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id - uses: ./.github/workflows/integration.yml + uses: ./.github/workflows/e2e.yml # Virtual job that can be configured as a required check before a PR can be merged. all-required-checks-done: name: All required checks done @@ -43,7 +43,7 @@ jobs: - test - codeql - build - - integration + - e2e runs-on: ubuntu-latest steps: - run: echo "All required checks done" diff --git a/.github/workflows/integration.yml b/.github/workflows/e2e.yml similarity index 99% rename from .github/workflows/integration.yml rename to .github/workflows/e2e.yml index 995e7f8..b400ff0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/e2e.yml @@ -1,4 +1,4 @@ -name: Integration +name: End2End tests on: workflow_call: From 561dced96666c2aaeb2019e7ae8ffa769b42bf35 Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Thu, 20 Mar 2025 15:25:56 +0100 Subject: [PATCH 4/6] Use armada-operator from sources in e2e workflow Signed-off-by: Enrico Minack --- .github/workflows/e2e.yml | 2 +- Makefile | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b400ff0..14d377d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -20,7 +20,7 @@ jobs: cache-prefix: go-integration-kind - name: Starting Armada cluster - run: make kind-all + run: make kind-all-local - name: Create queue run: | diff --git a/Makefile b/Makefile index 7ef57bd..d3bee9d 100644 --- a/Makefile +++ b/Makefile @@ -211,8 +211,12 @@ ifndef ignore-not-found ignore-not-found = false endif +.PHONY: kind-all kind-all: kind-create-cluster install-and-wait-cert-manager helm-repos helm-install install-armada-deps wait-for-armada-deps create-armada-namespace apply-armada-crs create-armadactl-config apply-default-priority-class get-armadactl ## Install everything +.PHONY: kind-all-local +kind-all-local: kind-create-cluster install-and-wait-cert-manager helm-repos helm-install-local install-armada-deps wait-for-armada-deps create-armada-namespace apply-armada-crs create-armadactl-config apply-default-priority-class get-armadactl ## Install everything from sources + .PHONY: kind-all-dev kind-all-dev: kind-create-cluster kind-deploy helm-repos install-armada-deps wait-for-armada-deps create-armada-namespace apply-armada-crs create-armadactl-config apply-default-priority-class get-armadactl ## Install everything with Operator built from scratch From c04d991183646ee14bfd85fe02ea4441e3369fa2 Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Thu, 20 Mar 2025 16:31:38 +0100 Subject: [PATCH 5/6] Move e2e test into scripts/e2e-test.sh Signed-off-by: Enrico Minack --- .github/workflows/e2e.yml | 76 ++------------------------------------- scripts/e2e-test.sh | 14 ++++++++ 2 files changed, 16 insertions(+), 74 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 14d377d..49a8633 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -19,77 +19,5 @@ jobs: with: cache-prefix: go-integration-kind - - name: Starting Armada cluster - run: make kind-all-local - - - name: Create queue - run: | - # Create queue - for second in {1..60} - do - if ./bin/app/armadactl create queue example 2> armadactl-${second}.log - then - if [[ second -eq 1 ]] - then - echo "Successfully created queue 'example'" - else - echo "Successfully created queue 'example' after ${second}s" - echo "Previously failed due to:" - cat armadactl-$((second-1)).log - fi - exit - fi - sleep 1 - done - - echo "Failed to create queue 'example' after ${second}s" - cat armadactl-${second}.log >&2 - exit 1 - - - name: Test queue exists - run: | - # Test queue exists - for second in {1..60} - do - if ./bin/app/armadactl get queue example 2> armadactl-${second}.log - then - if [[ second -eq 1 ]] - then - echo "Successfully inspected queue 'example'" - else - echo "Successfully inspected queue 'example' after ${second}s" - echo "Previously failed due to:" - cat armadactl-$((second-1)).log - fi - exit - fi - sleep 1 - done - - echo "Failed to inspected queue 'example' after ${second}s" - cat armadactl-${second}.log >&2 - exit 1 - - - name: Submit job - run: | - # Submit job - for second in {1..60} - do - if ./bin/app/armadactl submit dev/quickstart/example-job.yaml 2> armadactl-${second}.log - then - if [[ attempt -eq 1 ]] - then - echo "Successfully submitted job" - else - echo "Successfully submitted job after ${second}s" - echo "Previously failed due to:" - cat armadactl-$((second-1)).log - fi - exit - fi - sleep 1 - done - - echo "Failed to submitted job after ${second}s" - cat armadactl-${second}.log >&2 - exit 1 + - name: E2E test + run: ./scripts/e2e-test.sh diff --git a/scripts/e2e-test.sh b/scripts/e2e-test.sh index 80e1be2..44ebcb7 100755 --- a/scripts/e2e-test.sh +++ b/scripts/e2e-test.sh @@ -9,6 +9,14 @@ log() { echo -e "${GREEN}$1${NC}" } +armadactl-retry() { + for i in {1..60}; do + if ! armadactl "$@"; then + sleep 1 + fi + done +} + log "Running e2e tests..." log "Creating kind cluster..." @@ -30,5 +38,11 @@ done log "Waiting for Armada Operator pod to be ready..." kubectl wait --for='condition=Ready' pods -l 'app.kubernetes.io/name=armada-operator' --timeout=600s --namespace armada-system +log "Creating test queue..." +armadactl-retry create queue test + +log "Submitting job..." +armadactl-retry submit dev/quickstart/example-job.yaml + log "Deleting kind cluster..." make kind-delete-cluster From 61007463bdb21196538139832a49318962fae0d0 Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Fri, 21 Mar 2025 15:21:01 +0100 Subject: [PATCH 6/6] Rework e2e-test.sh --- scripts/e2e-test.sh | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/scripts/e2e-test.sh b/scripts/e2e-test.sh index 44ebcb7..526b96f 100755 --- a/scripts/e2e-test.sh +++ b/scripts/e2e-test.sh @@ -11,35 +11,22 @@ log() { armadactl-retry() { for i in {1..60}; do - if ! armadactl "$@"; then - sleep 1 + if armadactl "$@"; then + return 0 fi + sleep 1 done + echo "armadactl command failed after 60 attempts" >&2 + exit 1 } log "Running e2e tests..." -log "Creating kind cluster..." -make kind-create-cluster - -log "Installing cert-manager..." -make install-and-wait-cert-manager - -log "Building latest version of the operator and installing it via Helm..." -make helm-install-local - -for i in {1..5}; do - kubectl get pods -n armada-system - kubectl describe deployment armada-operator-controller-manager -n armada-system - sleep 15 - kubectl logs deployment/armada-operator-controller-manager -n armada-system -done - -log "Waiting for Armada Operator pod to be ready..." -kubectl wait --for='condition=Ready' pods -l 'app.kubernetes.io/name=armada-operator' --timeout=600s --namespace armada-system +log "Creating test environment cluster..." +make kind-all-local log "Creating test queue..." -armadactl-retry create queue test +armadactl-retry create queue example log "Submitting job..." armadactl-retry submit dev/quickstart/example-job.yaml