From 365561e42dbf35939b7d030c8aedea4672abd13f Mon Sep 17 00:00:00 2001 From: Andrew Melnick Date: Sat, 18 Mar 2023 16:36:24 -0600 Subject: [PATCH] Add automated cases to demonstrate archiving bug --- .gitignore | 2 + argo-workflows/values.archives.yaml | 34 +++++++++++++++++ argo-workflows/values.yaml | 29 --------------- down.sh | 3 ++ run-all.sh | 10 +++++ up-archives-after-workflow.sh | 57 +++++++++++++++++++++++++++++ up-no-archives.sh | 42 +++++++++++++++++++++ up-with-archives.sh | 44 ++++++++++++++++++++++ 8 files changed, 192 insertions(+), 29 deletions(-) create mode 100644 .gitignore create mode 100644 argo-workflows/values.archives.yaml create mode 100755 down.sh create mode 100755 run-all.sh create mode 100755 up-archives-after-workflow.sh create mode 100755 up-no-archives.sh create mode 100755 up-with-archives.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..205ff91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +wf-final*.yaml +*.swp diff --git a/argo-workflows/values.archives.yaml b/argo-workflows/values.archives.yaml new file mode 100644 index 0000000..8dcca8e --- /dev/null +++ b/argo-workflows/values.archives.yaml @@ -0,0 +1,34 @@ +argo-workflows: + controller: + workflowDefaults: + spec: + ttlStrategy: + secondsAfterCompletion: 84600 # 1 day + persistence: + archive: true + postgresql: + host: postgres.postgres.svc.cluster.local + port: 5432 + database: workflow + tableName: argo_workflows + userNameSecret: + name: argo-postgres-config + key: username + passwordSecret: + name: argo-postgres-config + key: password + ssl: false + sslmode: disable + useDefaultArtifactRepo: true + artifactRepository: + archiveLogs: true + s3: + bucket: workflows + endpoint: minio.minio.svc.cluster.local:9000 + insecure: true + accessKeySecret: + name: minio-creds + key: accesskey + secretKeySecret: + name: minio-creds + key: secretkey diff --git a/argo-workflows/values.yaml b/argo-workflows/values.yaml index 6547f62..00498ba 100644 --- a/argo-workflows/values.yaml +++ b/argo-workflows/values.yaml @@ -18,26 +18,9 @@ argo-workflows: workflowDefaults: spec: serviceAccountName: argo-workflow - ttlStrategy: - secondsAfterCompletion: 84600 # 1 day workflowNamespaces: - argo - default - persistence: - archive: true - postgresql: - host: postgres.postgres.svc.cluster.local - port: 5432 - database: workflow - tableName: argo_workflows - userNameSecret: - name: argo-postgres-config - key: username - passwordSecret: - name: argo-postgres-config - key: password - ssl: false - sslmode: disable executor: image: {} # registry: our-harbor-cache/quay.io @@ -79,18 +62,6 @@ argo-workflows: # rbac: # enabled: true useDefaultArtifactRepo: true - artifactRepository: - archiveLogs: true - s3: - bucket: workflows - endpoint: minio.minio.svc.cluster.local:9000 - insecure: true - accessKeySecret: - name: minio-creds - key: accesskey - secretKeySecret: - name: minio-creds - key: secretkey workflow: serviceAccount: create: true diff --git a/down.sh b/down.sh new file mode 100755 index 0000000..61db711 --- /dev/null +++ b/down.sh @@ -0,0 +1,3 @@ +#!/bin/bash -xeu + +k3d cluster delete workflow-archiving diff --git a/run-all.sh b/run-all.sh new file mode 100755 index 0000000..c1754ed --- /dev/null +++ b/run-all.sh @@ -0,0 +1,10 @@ +#!/bin/bash -xeu + +export BATCH_MODE=1 +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +"${SCRIPTPATH}/up-no-archives.sh" +"${SCRIPTPATH}/up-with-archives.sh" +"${SCRIPTPATH}/up-archives-after-workflow.sh" + +diff -y wf-final-archive.yaml wf-final-archive-after-workflow.yaml diff --git a/up-archives-after-workflow.sh b/up-archives-after-workflow.sh new file mode 100755 index 0000000..02c8d91 --- /dev/null +++ b/up-archives-after-workflow.sh @@ -0,0 +1,57 @@ +#!/bin/bash -xeu + +# Create cluster, install deps +k3d cluster create --config k3d.conf +kubectl get pods -Aw & +kubectl create namespace minio && kubectl -n minio apply -f minio +kubectl create namespace postgres && kubectl -n postgres apply -f postgres +kubectl create namespace argo +kubectl apply -n argo -f workflows-additions + +# Install Argo w/out archive +cd argo-workflows +helm dependency update +helm template -n argo argo-workflows -f values.yaml . > todeploy.yaml +kubectl apply -n argo -f todeploy.yaml +kubectl get workflows -Aw & +kubectl -n argo rollout status deployment/argo-workflows-workflow-controller +kubectl -n argo rollout status deployment/argo-workflows-server +cd .. + +# Create and wait for workflow to finish +wf_name=$(kubectl -n argo create -f hello-world/hello-world.yaml -o name | awk -F '/' '{ print $2 }') +argo -n argo wait "${wf_name}" + +# Add archive to Argo, wait for update +cd argo-workflows +helm dependency update +helm template -n argo argo-workflows -f values.yaml -f values.archives.yaml . > todeploy.yaml +kubectl apply -n argo -f todeploy.yaml +kubectl -n argo rollout restart deployment/argo-workflows-workflow-controller +kubectl -n argo rollout restart deployment/argo-workflows-server +kubectl -n argo rollout status deployment/argo-workflows-server +kubectl -n argo rollout status deployment/argo-workflows-workflow-controller +cd .. +sleep 3 +kubectl -n argo port-forward svc/argo-workflows-server 2746:2746 & +sleep 3 + +# Delete workflow, check archive +kubectl -n argo get workflow "${wf_name}" -o yaml | tee /dev/stderr > wf-final-archive-after-workflow.yaml +kubectl -n argo delete workflow "${wf_name}" +if ! ARGO_SERVER=localhost:2746 ARGO_TOKEN='BEARER foo' ARGO_INSECURE_SKIP_VERIFY=true argo -n argo archive get "${wf_name}"; then + echo 'Workflow was not archived!' +fi + +# Debugging +if [ -z "${BATCH_MODE:-}" ]; then + if which xdg-open; then + xdg-open https://localhost:2746/archived-workflows + elif which open; then + open https://localhost:2746/archived-workflows + fi + read -p "Paused, press enter to clean up cluster" +fi + +# Cleanup +k3d cluster delete workflow-archiving diff --git a/up-no-archives.sh b/up-no-archives.sh new file mode 100755 index 0000000..83b2c49 --- /dev/null +++ b/up-no-archives.sh @@ -0,0 +1,42 @@ +#!/bin/bash -xeu + +# Create cluster, install deps +k3d cluster create --config k3d.conf +kubectl get pods -Aw & +kubectl create namespace argo +kubectl apply -n argo -f workflows-additions + +# Install Argo w/out archive +cd argo-workflows +helm dependency update +helm template -n argo argo-workflows -f values.yaml . > todeploy.yaml +kubectl apply -n argo -f todeploy.yaml +kubectl get workflows -Aw & +kubectl -n argo rollout status deployment/argo-workflows-workflow-controller +kubectl -n argo rollout status deployment/argo-workflows-server +cd .. +kubectl -n argo port-forward --address 0.0.0.0 svc/argo-workflows-server 2746:2746 & + +# Create and wait for workflow to finish +wf_name=$(kubectl -n argo create -f hello-world/hello-world.yaml -o name | awk -F '/' '{ print $2 }') +argo -n argo wait "${wf_name}" + +# Delete workflow, check archive +kubectl -n argo get workflow "${wf_name}" -o yaml | tee /dev/stderr > wf-final-no-archive.yaml +kubectl -n argo delete workflow "${wf_name}" +if ! ARGO_SERVER=localhost:2746 ARGO_TOKEN='BEARER foo' ARGO_INSECURE_SKIP_VERIFY=true argo -n argo archive get "${wf_name}"; then + echo 'Workflow was not archived!' +fi + +# Debugging +if [ -z "${BATCH_MODE:-}" ]; then + if which xdg-open; then + xdg-open https://localhost:2746 + elif which open; then + open https://localhost:2746 + fi + read -p "Paused, press enter to clean up cluster" +fi + +# Cleanup +k3d cluster delete workflow-archiving diff --git a/up-with-archives.sh b/up-with-archives.sh new file mode 100755 index 0000000..86341a9 --- /dev/null +++ b/up-with-archives.sh @@ -0,0 +1,44 @@ +#!/bin/bash -xeu + +# Create cluster, install deps +k3d cluster create --config k3d.conf +kubectl get pods -Aw & +kubectl create namespace minio && kubectl -n minio apply -f minio +kubectl create namespace postgres && kubectl -n postgres apply -f postgres +kubectl create namespace argo +kubectl apply -n argo -f workflows-additions + +# Install Argo w/out archive +cd argo-workflows +helm dependency update +helm template -n argo argo-workflows -f values.yaml -f values.archives.yaml . > todeploy.yaml +kubectl apply -n argo -f todeploy.yaml +kubectl get workflows -Aw & +kubectl -n argo rollout status deployment/argo-workflows-workflow-controller +kubectl -n argo rollout status deployment/argo-workflows-server +cd .. +kubectl -n argo port-forward svc/argo-workflows-server 2746:2746 & + +# Create and wait for workflow to finish +wf_name=$(kubectl -n argo create -f hello-world/hello-world.yaml -o name | awk -F '/' '{ print $2 }') +argo -n argo wait "${wf_name}" + +# Delete workflow, check archive +kubectl -n argo get workflow "${wf_name}" -o yaml | tee /dev/stderr > wf-final-archive.yaml +kubectl -n argo delete workflow "${wf_name}" +if ! ARGO_SERVER=localhost:2746 ARGO_TOKEN='BEARER foo' ARGO_INSECURE_SKIP_VERIFY=true argo -n argo archive get "${wf_name}"; then + echo 'Workflow was not archived!' +fi + +# Debugging +if [ -z "${BATCH_MODE:-}" ]; then + if which xdg-open; then + xdg-open https://localhost:2746 + elif which open; then + open https://localhost:2746 + fi + read -p "Paused, press enter to clean up cluster" +fi + +# Cleanup +k3d cluster delete workflow-archiving