From 40dd26978decfa5ecd284e76932ee437928f24a9 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Tue, 18 May 2021 17:08:52 +0200 Subject: [PATCH 1/2] Implement CCCMO render to run CCM on bootstrap node This allows master nodes to be initialized with CCM replica from bootstrap, speeding up process of cluster installation. Without CCM running on bootstrap node, most of the cluster operator and operand pods would have to tolerate new taint and a different configuration of node.kubernetes.io/not-ready (NoSchedule) taint: - effect: NoSchedule key: node.cloudprovider.kubernetes.io/uninitialized value: "true" - effect: NoSchedule key: node.kubernetes.io/not-ready Without this bootstrap addition or taint toleration the cluster with CCM would never become ready. To test this feature on AWS build installer and deploy on AWS. To create a cluster run: ```bash ./hack/build.sh ./bin/openshift-install create manifests cat < manifests/manifest_feature_gate.yaml --- apiVersion: config.openshift.io/v1 kind: FeatureGate metadata: annotations: include.release.openshift.io/self-managed-high-availability: "true" include.release.openshift.io/single-node-developer: "true" release.openshift.io/create-only: "true" name: cluster spec: customNoUpgrade: enabled: - ExternalCloudProvider - CSIMigrationAWS - CSIMigrationOpenStack featureSet: CustomNoUpgrade EOF OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE=quay.io/dgrigore/release:cccmo-render ./bin/openshift-install create cluster ``` --- .../files/usr/local/bin/bootkube.sh.template | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index 32c012666a9..fcb76614fc0 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -51,6 +51,7 @@ CLUSTER_ETCD_OPERATOR_IMAGE=$(image_for cluster-etcd-operator) CONFIG_OPERATOR_IMAGE=$(image_for cluster-config-operator) KUBE_APISERVER_OPERATOR_IMAGE=$(image_for cluster-kube-apiserver-operator) KUBE_CONTROLLER_MANAGER_OPERATOR_IMAGE=$(image_for cluster-kube-controller-manager-operator) +CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE=$(image_for cluster-cloud-controller-manager-operator) KUBE_SCHEDULER_OPERATOR_IMAGE=$(image_for cluster-kube-scheduler-operator) INGRESS_OPERATOR_IMAGE=$(image_for cluster-ingress-operator) @@ -230,6 +231,36 @@ then record_service_stage_success fi +if [ ! -f cloud-controller-manager-bootstrap.done ] +then + record_service_stage_start "cluster-cloud-controller-manager-bootstrap" + echo "Rendering Cloud Controller Manager core manifests..." + + rm --recursive --force cloud-controller-manager-bootstrap + + # Copy the CCCMO images configMap to resolve pod images from internal registry in CCCMO render run + # Source config map is located in https://github.com/openshift/cluster-cloud-controller-manager-operator/blob/master/manifests/0000_26_cloud-controller-manager-operator_01_images.configmap.yaml + bootkube_podman_run \ + --name copy-cloud-controller-manager-images \ + --entrypoint /bin/bash "${RELEASE_IMAGE_DIGEST}" \ + -c "cat /release-manifests/0000_26_cloud-controller-manager-operator_01_images.configmap.yaml" > manifests/ccm-images.yaml + + bootkube_podman_run \ + --name cloud-controller-render \ + --volume "$PWD:/assets:z" \ + "${CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE}" \ + /render run \ + --images-file=/assets/manifests/ccm-images.yaml \ + --dest-dir=/assets/cloud-controller-manager-bootstrap \ + --cluster-infrastructure-file=/assets/manifests/cluster-infrastructure-02-config.yml + + # Copy rendered resources to manifests folder + cp cloud-controller-manager-bootstrap/manifests/* manifests/ + + touch cloud-controller-manager-bootstrap.done + record_service_stage_success +fi + if [ ! -f kube-scheduler-bootstrap.done ] then record_service_stage_start "kube-scheduler-bootstrap" From a45cffc97aeee510a35dabc1bc9b5f9c0cfa6e7f Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 7 Jun 2021 14:31:19 +0200 Subject: [PATCH 2/2] Add support for cloud-config-file flag to unpack and copy cloud-config in CCCMO --- .../files/usr/local/bin/bootkube.sh.template | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index fcb76614fc0..7e16d879b4a 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -245,6 +245,11 @@ then --entrypoint /bin/bash "${RELEASE_IMAGE_DIGEST}" \ -c "cat /release-manifests/0000_26_cloud-controller-manager-operator_01_images.configmap.yaml" > manifests/ccm-images.yaml + ADDITIONAL_FLAGS="" + if [ -f "$PWD/manifests/cloud-provider-config.yaml" ]; then + ADDITIONAL_FLAGS="--cloud-config-file=/assets/manifests/cloud-provider-config.yaml" + fi + bootkube_podman_run \ --name cloud-controller-render \ --volume "$PWD:/assets:z" \ @@ -252,10 +257,14 @@ then /render run \ --images-file=/assets/manifests/ccm-images.yaml \ --dest-dir=/assets/cloud-controller-manager-bootstrap \ - --cluster-infrastructure-file=/assets/manifests/cluster-infrastructure-02-config.yml + --cluster-infrastructure-file=/assets/manifests/cluster-infrastructure-02-config.yml \ + ${ADDITIONAL_FLAGS} # Copy rendered resources to manifests folder - cp cloud-controller-manager-bootstrap/manifests/* manifests/ + cp -r cloud-controller-manager-bootstrap/bootstrap/. bootstrap-manifests/ + # Copy cloud config to /etc/kubernetes/bootstrap-configs + cp -r cloud-controller-manager-bootstrap/config/. /etc/kubernetes/bootstrap-configs + touch cloud-controller-manager-bootstrap.done record_service_stage_success