diff --git a/cinder/templates/bin/_ceph-keyring.sh.tpl b/cinder/templates/bin/_ceph-keyring.sh.tpl index 521a74848c..3574831e12 100644 --- a/cinder/templates/bin/_ceph-keyring.sh.tpl +++ b/cinder/templates/bin/_ceph-keyring.sh.tpl @@ -17,16 +17,22 @@ limitations under the License. set -ex export HOME=/tmp -cat < /etc/ceph/ceph.client.${RBD_USER}.keyring -[client.${RBD_USER}] - key = $(cat /tmp/client-keyring) -EOF - {{- if and .Values.ceph_client.enable_external_ceph_backend .Values.ceph_client.external_ceph.rbd_user }} -cat < /etc/ceph/ceph.client.${EXTERNAL_RBD_USER}.keyring +# Handle external Ceph keyring +if [ -n "${EXTERNAL_RBD_USER}" ] && [ -f /tmp/external-ceph-client-keyring ]; then + cat < /etc/ceph/ceph.client.${EXTERNAL_RBD_USER}.keyring [client.${EXTERNAL_RBD_USER}] key = $(cat /tmp/external-ceph-client-keyring) EOF +fi {{- end }} +# Handle internal Ceph keyring (only if mounted) +if [ -n "${RBD_USER}" ] && [ -f /tmp/client-keyring ]; then + cat < /etc/ceph/ceph.client.${RBD_USER}.keyring +[client.${RBD_USER}] + key = $(cat /tmp/client-keyring) +EOF +fi + exit 0 diff --git a/cinder/templates/deployment-backup.yaml b/cinder/templates/deployment-backup.yaml index f45f583a85..d46270f9ce 100644 --- a/cinder/templates/deployment-backup.yaml +++ b/cinder/templates/deployment-backup.yaml @@ -93,18 +93,28 @@ spec: {{ end }} {{- range $name := rest (splitList "," (include "cinder.utils.ceph_backend_list" $envAll)) }} {{- $backend := index $envAll.Values.conf.backends $name }} - {{- if eq $internal_ceph_backend $name }} + {{- $is_internal := eq $internal_ceph_backend $name }} + {{- $is_external := and $envAll.Values.ceph_client.enable_external_ceph_backend (eq $backend.rbd_ceph_conf "/etc/ceph/external-ceph.conf") }} + {{- if or $is_internal $is_external }} - name: ceph-keyring-placement-{{ $name | lower }} {{ tuple $envAll "cinder_backup" | include "helm-toolkit.snippets.image" | indent 10 }} {{ dict "envAll" $envAll "application" "cinder_backup" "container" "ceph_keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} command: - /tmp/ceph-keyring.sh env: + {{- if $is_internal }} - name: RBD_USER value: {{ $backend.rbd_user | quote }} + {{- end }} {{- if and $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }} + {{- if $is_external }} + - name: RBD_USER + value: {{ $envAll.Values.ceph_client.external_ceph.rbd_user | quote }} + {{- end }} + {{- if or $is_internal $is_external }} - name: EXTERNAL_RBD_USER value: {{ $envAll.Values.ceph_client.external_ceph.rbd_user | quote }} + {{- end }} {{- end }} volumeMounts: - name: pod-tmp @@ -115,11 +125,19 @@ spec: mountPath: /tmp/ceph-keyring.sh subPath: ceph-keyring.sh readOnly: true + {{- if $is_internal }} - name: ceph-keyring mountPath: /tmp/client-keyring subPath: key readOnly: true - {{- if and $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }} + {{- end }} + {{- if and $is_external $envAll.Values.ceph_client.external_ceph.rbd_user }} + - name: external-ceph-keyring + mountPath: /tmp/external-ceph-client-keyring + subPath: key + readOnly: true + {{- end }} + {{- if and $is_internal $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }} - name: external-ceph-keyring mountPath: /tmp/external-ceph-client-keyring subPath: key @@ -188,10 +206,12 @@ spec: - name: etcceph mountPath: /etc/ceph {{- if not .Values.backup.external_ceph_rbd.enabled }} + {{- if eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll) }} - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true + {{- end }} {{- else if .Values.backup.external_ceph_rbd.configmap }} - name: external-backup-ceph-etc mountPath: /etc/ceph/ceph.conf @@ -204,11 +224,13 @@ spec: readOnly: true {{- end }} {{- if (contains "cinder.backup.drivers.ceph" .Values.conf.cinder.DEFAULT.backup_driver) }} + {{- if not .Values.backup.external_ceph_rbd.enabled }} - name: ceph-backup-keyring mountPath: /tmp/client-keyring subPath: key readOnly: true - {{- else }} + {{- end }} + {{- else if eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll) }} - name: ceph-keyring mountPath: /tmp/client-keyring subPath: key @@ -312,10 +334,12 @@ spec: name: {{ .Values.backup.external_ceph_rbd.configmap }} defaultMode: 0444 {{- end }} + {{- if eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll) }} - name: ceph-etc configMap: name: {{ .Values.ceph_client.configmap }} defaultMode: 0444 + {{- end }} {{- if and .Values.ceph_client.enable_external_ceph_backend .Values.ceph_client.external_ceph.configmap }} - name: external-ceph-etc configMap: @@ -324,11 +348,13 @@ spec: {{- end }} {{ end }} {{- if (contains "cinder.backup.drivers.ceph" .Values.conf.cinder.DEFAULT.backup_driver) }} + {{- if not .Values.backup.external_ceph_rbd.enabled }} - name: ceph-backup-keyring secret: secretName: {{ .Values.secrets.rbd.backup | quote }} + {{- end }} {{ end }} - {{- if eq "true" (include "cinder.utils.has_ceph_backend" $envAll) }} + {{- if and (eq "true" (include "cinder.utils.has_ceph_backend" $envAll)) (eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll)) }} - name: ceph-keyring secret: secretName: {{ .Values.secrets.rbd.volume | quote }} diff --git a/cinder/templates/deployment-volume.yaml b/cinder/templates/deployment-volume.yaml index d1cd918866..6e058a3e3d 100644 --- a/cinder/templates/deployment-volume.yaml +++ b/cinder/templates/deployment-volume.yaml @@ -70,18 +70,28 @@ spec: {{ tuple $envAll "volume" $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} {{- range $name := rest (splitList "," (include "cinder.utils.ceph_backend_list" $envAll)) }} {{- $backend := index $envAll.Values.conf.backends $name }} - {{- if eq $internal_ceph_backend $name }} + {{- $is_internal := eq $internal_ceph_backend $name }} + {{- $is_external := and $envAll.Values.ceph_client.enable_external_ceph_backend (eq $backend.rbd_ceph_conf "/etc/ceph/external-ceph.conf") }} + {{- if or $is_internal $is_external }} - name: ceph-keyring-placement-{{ $name | lower }} {{ tuple $envAll "cinder_volume" | include "helm-toolkit.snippets.image" | indent 10 }} {{ dict "envAll" $envAll "application" "cinder_volume" "container" "ceph_keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} command: - /tmp/ceph-keyring.sh env: + {{- if $is_internal }} - name: RBD_USER value: {{ $backend.rbd_user | quote }} + {{- end }} {{- if and $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }} + {{- if $is_external }} + - name: RBD_USER + value: {{ $envAll.Values.ceph_client.external_ceph.rbd_user | quote }} + {{- end }} + {{- if or $is_internal $is_external }} - name: EXTERNAL_RBD_USER value: {{ $envAll.Values.ceph_client.external_ceph.rbd_user | quote }} + {{- end }} {{- end }} volumeMounts: - name: pod-tmp @@ -92,11 +102,19 @@ spec: mountPath: /tmp/ceph-keyring.sh subPath: ceph-keyring.sh readOnly: true + {{- if $is_internal }} - name: ceph-keyring mountPath: /tmp/client-keyring subPath: key readOnly: true - {{- if and $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }} + {{- end }} + {{- if and $is_external $envAll.Values.ceph_client.external_ceph.rbd_user }} + - name: external-ceph-keyring + mountPath: /tmp/external-ceph-client-keyring + subPath: key + readOnly: true + {{- end }} + {{- if and $is_internal $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }} - name: external-ceph-keyring mountPath: /tmp/external-ceph-client-keyring subPath: key @@ -189,6 +207,7 @@ spec: {{- if eq "true" (include "cinder.utils.has_ceph_backend" $envAll) }} - name: etcceph mountPath: /etc/ceph + {{- if eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll) }} - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf @@ -197,6 +216,7 @@ spec: mountPath: /tmp/client-keyring subPath: key readOnly: true + {{- end }} {{- if .Values.ceph_client.enable_external_ceph_backend }} {{- if .Values.ceph_client.external_ceph.configmap }} - name: external-ceph-etc @@ -318,25 +338,27 @@ spec: {{- if eq "true" (include "cinder.utils.has_ceph_backend" $envAll) }} - name: etcceph emptyDir: {} + {{- if eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll) }} - name: ceph-etc configMap: name: {{ .Values.ceph_client.configmap }} defaultMode: 0444 + - name: ceph-keyring + secret: + secretName: {{ .Values.secrets.rbd.volume | quote }} + {{- end }} {{- if and .Values.ceph_client.enable_external_ceph_backend .Values.ceph_client.external_ceph.configmap }} - name: external-ceph-etc configMap: name: {{ .Values.ceph_client.external_ceph.configmap }} defaultMode: 0444 {{- end }} - - name: ceph-keyring - secret: - secretName: {{ .Values.secrets.rbd.volume | quote }} {{- if and .Values.ceph_client.enable_external_ceph_backend .Values.ceph_client.external_ceph.rbd_user }} - name: external-ceph-keyring secret: secretName: {{ .Values.secrets.rbd.volume_external | quote }} - {{ end }} - {{ end }} + {{- end }} + {{- end }} {{- if eq ( split "://" .Values.conf.cinder.coordination.backend_url )._0 "file" }} # NOTE (portdirect): this will need to be set to a shared mount amongst all cinder # pods for the coordination backend to be fully functional. diff --git a/cinder/templates/utils/_has_internal_ceph_backend.tpl b/cinder/templates/utils/_has_internal_ceph_backend.tpl new file mode 100644 index 0000000000..9c30c78b12 --- /dev/null +++ b/cinder/templates/utils/_has_internal_ceph_backend.tpl @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "cinder.utils.has_internal_ceph_backend" -}} + {{- $has_internal := false -}} + {{- range $_, $backend := .Values.conf.backends -}} + {{- if kindIs "map" $backend -}} + {{- if and (eq $backend.volume_driver "cinder.volume.drivers.rbd.RBDDriver") (ne $backend.rbd_ceph_conf "/etc/ceph/external-ceph.conf") -}} + {{- $has_internal = true -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- $has_internal -}} +{{- end -}} + diff --git a/cinder/values.yaml b/cinder/values.yaml index e5635d4aff..04a30be0b3 100644 --- a/cinder/values.yaml +++ b/cinder/values.yaml @@ -850,7 +850,11 @@ conf: # via the endpoints section. osapi_volume_listen_port: null enabled_backends: "rbd1" + # When using external Ceph, add the external backend name here, e.g.: + # enabled_backends: "rbd1,external_rbd" default_volume_type: "rbd1" + # When using external Ceph as default, change this to the external backend name, e.g.: + # default_volume_type: "external_rbd" # NOTE(portdirect): "cinder.backup.drivers.ceph" and # "cinder.backup.drivers.posix" also supported # NOTE(rchurch): As of Stein, drivers by class name are required