From 8e56e39dad301d1f9c50d398f9779fc7f042b6bd Mon Sep 17 00:00:00 2001 From: Denis Moiseev Date: Wed, 14 Jul 2021 11:23:37 +0200 Subject: [PATCH 1/3] Enable e2e tests with CCM functionality for Azure platform - e2e tests workflow - install with enabled featuregate (optional) --- ...ud-controller-manager-operator-master.yaml | 8 ++ ...er-manager-operator-master-presubmits.yaml | 133 ++++++++++++++++++ .../ccm-conf-apply-feature-gate-commands.sh | 2 + ...m-conf-pre-create-feature-gate-commands.sh | 2 + .../openshift/e2e/azure/ccm/OWNERS | 16 +++ .../openshift/e2e/azure/ccm/install/OWNERS | 16 +++ ...e-azure-ccm-install-workflow.metadata.json | 23 +++ ...nshift-e2e-azure-ccm-install-workflow.yaml | 16 +++ ...shift-e2e-azure-ccm-workflow.metadata.json | 23 +++ .../ccm/openshift-e2e-azure-ccm-workflow.yaml | 23 +++ 10 files changed, 262 insertions(+) create mode 100644 ci-operator/step-registry/openshift/e2e/azure/ccm/OWNERS create mode 100644 ci-operator/step-registry/openshift/e2e/azure/ccm/install/OWNERS create mode 100644 ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.metadata.json create mode 100644 ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.yaml create mode 100644 ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.metadata.json create mode 100644 ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.yaml diff --git a/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master.yaml b/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master.yaml index 7a102e68f72fd..45ecb5c4d08e1 100644 --- a/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master.yaml +++ b/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master.yaml @@ -78,6 +78,14 @@ tests: steps: cluster_profile: aws workflow: openshift-e2e-aws-ccm-install +- as: e2e-azure-ccm + steps: + cluster_profile: azure4 + workflow: openshift-e2e-azure-ccm +- as: e2e-azure-ccm-install + steps: + cluster_profile: azure4 + workflow: openshift-e2e-azure-ccm-install zz_generated_metadata: branch: master org: openshift diff --git a/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master-presubmits.yaml b/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master-presubmits.yaml index a8bc2b789d6be..2c8515cc9b163 100644 --- a/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master-presubmits.yaml +++ b/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master-presubmits.yaml @@ -265,6 +265,139 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-aws-upgrade,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - master + cluster: build02 + context: ci/prow/e2e-azure-ccm + decorate: true + decoration_config: + skip_cloning: true + labels: + ci-operator.openshift.io/prowgen-controlled: "true" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-cluster-cloud-controller-manager-operator-master-e2e-azure-ccm + rerun_command: /test e2e-azure-ccm + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/usr/local/e2e-azure-ccm-cluster-profile + - --target=e2e-azure-ccm + command: + - ci-operator + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /usr/local/e2e-azure-ccm-cluster-profile + name: cluster-profile + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: cluster-profile + projected: + sources: + - secret: + name: cluster-secrets-azure4 + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-azure-ccm,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - master + cluster: build02 + context: ci/prow/e2e-azure-ccm-install + decorate: true + decoration_config: + skip_cloning: true + labels: + ci-operator.openshift.io/prowgen-controlled: "true" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-cluster-cloud-controller-manager-operator-master-e2e-azure-ccm-install + optional: true + rerun_command: /test e2e-azure-ccm-install + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/usr/local/e2e-azure-ccm-install-cluster-profile + - --target=e2e-azure-ccm-install + command: + - ci-operator + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /usr/local/e2e-azure-ccm-install-cluster-profile + name: cluster-profile + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: cluster-profile + projected: + sources: + - secret: + name: cluster-secrets-azure4 + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-azure-ccm-install,?($|\s.*) - agent: kubernetes always_run: false branches: diff --git a/ci-operator/step-registry/ccm/conf/apply-feature-gate/ccm-conf-apply-feature-gate-commands.sh b/ci-operator/step-registry/ccm/conf/apply-feature-gate/ccm-conf-apply-feature-gate-commands.sh index da20bbedf4c03..e9fc9e86fdf06 100644 --- a/ci-operator/step-registry/ccm/conf/apply-feature-gate/ccm-conf-apply-feature-gate-commands.sh +++ b/ci-operator/step-registry/ccm/conf/apply-feature-gate/ccm-conf-apply-feature-gate-commands.sh @@ -71,6 +71,8 @@ spec: - ExternalCloudProvider - CSIMigrationAWS - CSIMigrationOpenStack + - CSIMigrationAzureDisk + - CSIDriverAzureDisk featureSet: CustomNoUpgrade EOF } diff --git a/ci-operator/step-registry/ccm/conf/pre-create-feature-gate/ccm-conf-pre-create-feature-gate-commands.sh b/ci-operator/step-registry/ccm/conf/pre-create-feature-gate/ccm-conf-pre-create-feature-gate-commands.sh index 97aa3686dfa52..a52a53d88d575 100644 --- a/ci-operator/step-registry/ccm/conf/pre-create-feature-gate/ccm-conf-pre-create-feature-gate-commands.sh +++ b/ci-operator/step-registry/ccm/conf/pre-create-feature-gate/ccm-conf-pre-create-feature-gate-commands.sh @@ -17,5 +17,7 @@ spec: - ExternalCloudProvider - CSIMigrationAWS - CSIMigrationOpenStack + - CSIMigrationAzureDisk + - CSIDriverAzureDisk featureSet: CustomNoUpgrade EOF diff --git a/ci-operator/step-registry/openshift/e2e/azure/ccm/OWNERS b/ci-operator/step-registry/openshift/e2e/azure/ccm/OWNERS new file mode 100644 index 0000000000000..3fb9f70a534e1 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azure/ccm/OWNERS @@ -0,0 +1,16 @@ +approvers: +- danil-grigorev +- elmiko +- fedosin +- joelspeed +- mandre +- mdbooth +- lobziik +reviewers: +- danil-grigorev +- elmiko +- fedosin +- joelspeed +- mandre +- mdbooth +- lobziik diff --git a/ci-operator/step-registry/openshift/e2e/azure/ccm/install/OWNERS b/ci-operator/step-registry/openshift/e2e/azure/ccm/install/OWNERS new file mode 100644 index 0000000000000..3fb9f70a534e1 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azure/ccm/install/OWNERS @@ -0,0 +1,16 @@ +approvers: +- danil-grigorev +- elmiko +- fedosin +- joelspeed +- mandre +- mdbooth +- lobziik +reviewers: +- danil-grigorev +- elmiko +- fedosin +- joelspeed +- mandre +- mdbooth +- lobziik diff --git a/ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.metadata.json b/ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.metadata.json new file mode 100644 index 0000000000000..72e09e114f9ef --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.metadata.json @@ -0,0 +1,23 @@ +{ + "path": "openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.yaml", + "owners": { + "approvers": [ + "danil-grigorev", + "elmiko", + "fedosin", + "joelspeed", + "mandre", + "mdbooth", + "lobziik" + ], + "reviewers": [ + "danil-grigorev", + "elmiko", + "fedosin", + "joelspeed", + "mandre", + "mdbooth", + "lobziik" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.yaml b/ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.yaml new file mode 100644 index 0000000000000..4a805a72fdd80 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.yaml @@ -0,0 +1,16 @@ +workflow: + as: openshift-e2e-azure-ccm-install + steps: + pre: + - chain: ipi-conf-azure + - ref: ccm-conf-pre-create-feature-gate + - chain: ipi-install + test: + - ref: openshift-e2e-test + post: + - chain: gather-core-dump + - chain: ipi-azure-post + env: + TEST_CSI_DRIVER_MANIFEST: manifest-aws-ebs.yaml + documentation: |- + The Openshift E2E Azure CCM install workflow executes the common end-to-end test suite on Azure with a cluster configuration including CCM pods diff --git a/ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.metadata.json b/ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.metadata.json new file mode 100644 index 0000000000000..47248c935796c --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.metadata.json @@ -0,0 +1,23 @@ +{ + "path": "openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.yaml", + "owners": { + "approvers": [ + "danil-grigorev", + "elmiko", + "fedosin", + "joelspeed", + "mandre", + "mdbooth", + "lobziik" + ], + "reviewers": [ + "danil-grigorev", + "elmiko", + "fedosin", + "joelspeed", + "mandre", + "mdbooth", + "lobziik" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.yaml b/ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.yaml new file mode 100644 index 0000000000000..6ccfbf63528d7 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.yaml @@ -0,0 +1,23 @@ +workflow: + as: openshift-e2e-azure-ccm + steps: + pre: + - chain: ipi-azure-pre + - ref: ccm-conf-apply-feature-gate + - ref: storage-conf-csi-azure-disk + - ref: storage-conf-wait-for-csi-driver + - ref: storage-conf-wait-for-csi-migration + test: + - ref: openshift-e2e-test + post: + - chain: gather-core-dump + - chain: ipi-azure-post + env: + # for storage-conf-wait-for-csi-driver + CLUSTERCSIDRIVER: disk.csi.azure.com + TRUECONDITIONS: AzureDiskDriverControllerServiceControllerAvailable AzureDiskDriverNodeServiceControllerAvailable + TEST_CSI_DRIVER_MANIFEST: manifest-azure-disk.yaml + TEST_SUITE: openshift/csi + documentation: |- + The Openshift E2E Azure workflow using CCM as a primary mean to initialize nodes. + Executes the common end-to-end test suite on Azure to test updated cluster configuration. From cdf3abaaec5b13cabaa76aadf998fac4e279f6a9 Mon Sep 17 00:00:00 2001 From: Denis Moiseev Date: Tue, 20 Jul 2021 14:38:41 +0200 Subject: [PATCH 2/3] disable azure-ccm-install job --- ...ud-controller-manager-operator-master.yaml | 4 -- ...er-manager-operator-master-presubmits.yaml | 69 +------------------ 2 files changed, 1 insertion(+), 72 deletions(-) diff --git a/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master.yaml b/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master.yaml index 45ecb5c4d08e1..c5e48485c9a88 100644 --- a/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master.yaml +++ b/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master.yaml @@ -82,10 +82,6 @@ tests: steps: cluster_profile: azure4 workflow: openshift-e2e-azure-ccm -- as: e2e-azure-ccm-install - steps: - cluster_profile: azure4 - workflow: openshift-e2e-azure-ccm-install zz_generated_metadata: branch: master org: openshift diff --git a/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master-presubmits.yaml b/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master-presubmits.yaml index 2c8515cc9b163..02b2d812b883e 100644 --- a/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master-presubmits.yaml +++ b/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-master-presubmits.yaml @@ -269,7 +269,7 @@ presubmits: always_run: true branches: - master - cluster: build02 + cluster: build01 context: ci/prow/e2e-azure-ccm decorate: true decoration_config: @@ -331,73 +331,6 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-azure-ccm,?($|\s.*) - - agent: kubernetes - always_run: false - branches: - - master - cluster: build02 - context: ci/prow/e2e-azure-ccm-install - decorate: true - decoration_config: - skip_cloning: true - labels: - ci-operator.openshift.io/prowgen-controlled: "true" - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: pull-ci-openshift-cluster-cloud-controller-manager-operator-master-e2e-azure-ccm-install - optional: true - rerun_command: /test e2e-azure-ccm-install - spec: - containers: - - args: - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --lease-server-credentials-file=/etc/boskos/credentials - - --report-credentials-file=/etc/report/credentials - - --secret-dir=/usr/local/e2e-azure-ccm-install-cluster-profile - - --target=e2e-azure-ccm-install - command: - - ci-operator - image: ci-operator:latest - imagePullPolicy: Always - name: "" - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/boskos - name: boskos - readOnly: true - - mountPath: /usr/local/e2e-azure-ccm-install-cluster-profile - name: cluster-profile - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: boskos - secret: - items: - - key: credentials - path: credentials - secretName: boskos-credentials - - name: cluster-profile - projected: - sources: - - secret: - name: cluster-secrets-azure4 - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator - trigger: (?m)^/test( | .* )e2e-azure-ccm-install,?($|\s.*) - agent: kubernetes always_run: false branches: From 6b2a97f2563fc55df38c9ecee0fbc49e8d0468b1 Mon Sep 17 00:00:00 2001 From: Denis Moiseev Date: Tue, 20 Jul 2021 15:08:32 +0200 Subject: [PATCH 3/3] remove mandre and mbooth from azure workflows owners --- ci-operator/step-registry/openshift/e2e/azure/ccm/OWNERS | 4 ---- .../step-registry/openshift/e2e/azure/ccm/install/OWNERS | 4 ---- .../openshift-e2e-azure-ccm-install-workflow.metadata.json | 4 ---- .../azure/ccm/openshift-e2e-azure-ccm-workflow.metadata.json | 4 ---- 4 files changed, 16 deletions(-) diff --git a/ci-operator/step-registry/openshift/e2e/azure/ccm/OWNERS b/ci-operator/step-registry/openshift/e2e/azure/ccm/OWNERS index 3fb9f70a534e1..a81df72cdd2c9 100644 --- a/ci-operator/step-registry/openshift/e2e/azure/ccm/OWNERS +++ b/ci-operator/step-registry/openshift/e2e/azure/ccm/OWNERS @@ -3,14 +3,10 @@ approvers: - elmiko - fedosin - joelspeed -- mandre -- mdbooth - lobziik reviewers: - danil-grigorev - elmiko - fedosin - joelspeed -- mandre -- mdbooth - lobziik diff --git a/ci-operator/step-registry/openshift/e2e/azure/ccm/install/OWNERS b/ci-operator/step-registry/openshift/e2e/azure/ccm/install/OWNERS index 3fb9f70a534e1..a81df72cdd2c9 100644 --- a/ci-operator/step-registry/openshift/e2e/azure/ccm/install/OWNERS +++ b/ci-operator/step-registry/openshift/e2e/azure/ccm/install/OWNERS @@ -3,14 +3,10 @@ approvers: - elmiko - fedosin - joelspeed -- mandre -- mdbooth - lobziik reviewers: - danil-grigorev - elmiko - fedosin - joelspeed -- mandre -- mdbooth - lobziik diff --git a/ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.metadata.json b/ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.metadata.json index 72e09e114f9ef..498040befe318 100644 --- a/ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.metadata.json +++ b/ci-operator/step-registry/openshift/e2e/azure/ccm/install/openshift-e2e-azure-ccm-install-workflow.metadata.json @@ -6,8 +6,6 @@ "elmiko", "fedosin", "joelspeed", - "mandre", - "mdbooth", "lobziik" ], "reviewers": [ @@ -15,8 +13,6 @@ "elmiko", "fedosin", "joelspeed", - "mandre", - "mdbooth", "lobziik" ] } diff --git a/ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.metadata.json b/ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.metadata.json index 47248c935796c..22561100a4c6e 100644 --- a/ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.metadata.json +++ b/ci-operator/step-registry/openshift/e2e/azure/ccm/openshift-e2e-azure-ccm-workflow.metadata.json @@ -6,8 +6,6 @@ "elmiko", "fedosin", "joelspeed", - "mandre", - "mdbooth", "lobziik" ], "reviewers": [ @@ -15,8 +13,6 @@ "elmiko", "fedosin", "joelspeed", - "mandre", - "mdbooth", "lobziik" ] }