feat: support solo external ceph

cinder/templates/bin/_ceph-keyring.sh.tpl

@@ -17,16 +17,22 @@ limitations under the License.
 set -ex
 export HOME=/tmp
 
-cat <<EOF > /etc/ceph/ceph.client.${RBD_USER}.keyring
-[client.${RBD_USER}]
-    key = $(cat /tmp/client-keyring)
-EOF
-
 {{- if and .Values.ceph_client.enable_external_ceph_backend .Values.ceph_client.external_ceph.rbd_user }}
-cat <<EOF > /etc/ceph/ceph.client.${EXTERNAL_RBD_USER}.keyring
+# Handle external Ceph keyring
+if [ -n "${EXTERNAL_RBD_USER}" ] && [ -f /tmp/external-ceph-client-keyring ]; then
+  cat <<EOF > /etc/ceph/ceph.client.${EXTERNAL_RBD_USER}.keyring
 [client.${EXTERNAL_RBD_USER}]
     key = $(cat /tmp/external-ceph-client-keyring)
 EOF
+fi
 {{- end }}
 
+# Handle internal Ceph keyring (only if mounted)
+if [ -n "${RBD_USER}" ] && [ -f /tmp/client-keyring ]; then
+  cat <<EOF > /etc/ceph/ceph.client.${RBD_USER}.keyring
+[client.${RBD_USER}]
+    key = $(cat /tmp/client-keyring)
+EOF
+fi
+
 exit 0

cinder/templates/deployment-backup.yaml

@@ -93,18 +93,28 @@ spec:
         {{ end }}
         {{- range $name := rest (splitList "," (include "cinder.utils.ceph_backend_list" $envAll)) }}
           {{- $backend := index $envAll.Values.conf.backends $name }}
-            {{- if eq $internal_ceph_backend $name }}
+          {{- $is_internal := eq $internal_ceph_backend $name }}
+          {{- $is_external := and $envAll.Values.ceph_client.enable_external_ceph_backend (eq $backend.rbd_ceph_conf "/etc/ceph/external-ceph.conf") }}
+          {{- if or $is_internal $is_external }}
         - name: ceph-keyring-placement-{{ $name | lower }}
 {{ tuple $envAll "cinder_backup" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ dict "envAll" $envAll "application" "cinder_backup" "container" "ceph_keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
             - /tmp/ceph-keyring.sh
           env:
+            {{- if $is_internal }}
             - name: RBD_USER
               value: {{ $backend.rbd_user | quote }}
+            {{- end }}
 {{- if and $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }}
+            {{- if $is_external }}
+            - name: RBD_USER
+              value: {{ $envAll.Values.ceph_client.external_ceph.rbd_user | quote }}
+            {{- end }}
+            {{- if or $is_internal $is_external }}
             - name: EXTERNAL_RBD_USER
               value: {{ $envAll.Values.ceph_client.external_ceph.rbd_user | quote }}
+            {{- end }}
 {{- end }}
           volumeMounts:
             - name: pod-tmp
@@ -115,11 +125,19 @@ spec:
               mountPath: /tmp/ceph-keyring.sh
               subPath: ceph-keyring.sh
               readOnly: true
+            {{- if $is_internal }}
             - name: ceph-keyring
               mountPath: /tmp/client-keyring
               subPath: key
               readOnly: true
-           {{- if and $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }}
+            {{- end }}
+            {{- if and $is_external $envAll.Values.ceph_client.external_ceph.rbd_user }}
+            - name: external-ceph-keyring
+              mountPath: /tmp/external-ceph-client-keyring
+              subPath: key
+              readOnly: true
+            {{- end }}
+            {{- if and $is_internal $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }}
             - name: external-ceph-keyring
               mountPath: /tmp/external-ceph-client-keyring
               subPath: key
@@ -188,10 +206,12 @@ spec:
             - name: etcceph
               mountPath: /etc/ceph
             {{- if not .Values.backup.external_ceph_rbd.enabled }}
+            {{- if eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll) }}
             - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
+            {{- end }}
             {{- else if .Values.backup.external_ceph_rbd.configmap }}
             - name: external-backup-ceph-etc
               mountPath: /etc/ceph/ceph.conf
@@ -204,11 +224,13 @@ spec:
               readOnly: true
             {{- end }}
             {{- if (contains "cinder.backup.drivers.ceph" .Values.conf.cinder.DEFAULT.backup_driver) }}
+            {{- if not .Values.backup.external_ceph_rbd.enabled }}
             - name: ceph-backup-keyring
               mountPath: /tmp/client-keyring
               subPath: key
               readOnly: true
-            {{- else }}
+            {{- end }}
+            {{- else if eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll) }}
             - name: ceph-keyring
               mountPath: /tmp/client-keyring
               subPath: key
@@ -312,10 +334,12 @@ spec:
             name: {{ .Values.backup.external_ceph_rbd.configmap }}
             defaultMode: 0444
         {{- end }}
+        {{- if eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll) }}
         - name: ceph-etc
           configMap:
             name: {{ .Values.ceph_client.configmap }}
             defaultMode: 0444
+        {{- end }}
         {{- if and .Values.ceph_client.enable_external_ceph_backend .Values.ceph_client.external_ceph.configmap }}
         - name: external-ceph-etc
           configMap:
@@ -324,11 +348,13 @@ spec:
         {{- end }}
         {{ end }}
         {{- if (contains "cinder.backup.drivers.ceph" .Values.conf.cinder.DEFAULT.backup_driver) }}
+        {{- if not .Values.backup.external_ceph_rbd.enabled }}
         - name: ceph-backup-keyring
           secret:
             secretName: {{ .Values.secrets.rbd.backup | quote }}
+        {{- end }}
         {{ end }}
-        {{- if eq "true" (include "cinder.utils.has_ceph_backend" $envAll) }}
+        {{- if and (eq "true" (include "cinder.utils.has_ceph_backend" $envAll)) (eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll)) }}
         - name: ceph-keyring
           secret:
             secretName: {{ .Values.secrets.rbd.volume | quote }}

cinder/templates/deployment-volume.yaml

@@ -70,18 +70,28 @@ spec:
 {{ tuple $envAll "volume" $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
         {{- range $name := rest (splitList "," (include "cinder.utils.ceph_backend_list" $envAll)) }}
           {{- $backend := index $envAll.Values.conf.backends $name }}
-            {{- if eq $internal_ceph_backend $name }}
+          {{- $is_internal := eq $internal_ceph_backend $name }}
+          {{- $is_external := and $envAll.Values.ceph_client.enable_external_ceph_backend (eq $backend.rbd_ceph_conf "/etc/ceph/external-ceph.conf") }}
+          {{- if or $is_internal $is_external }}
         - name: ceph-keyring-placement-{{ $name | lower }}
 {{ tuple $envAll "cinder_volume" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ dict "envAll" $envAll "application" "cinder_volume" "container" "ceph_keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           command:
             - /tmp/ceph-keyring.sh
           env:
+            {{- if $is_internal }}
             - name: RBD_USER
               value: {{ $backend.rbd_user | quote }}
+            {{- end }}
 {{- if and $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }}
+            {{- if $is_external }}
+            - name: RBD_USER
+              value: {{ $envAll.Values.ceph_client.external_ceph.rbd_user | quote }}
+            {{- end }}
+            {{- if or $is_internal $is_external }}
             - name: EXTERNAL_RBD_USER
               value: {{ $envAll.Values.ceph_client.external_ceph.rbd_user | quote }}
+            {{- end }}
 {{- end }}
           volumeMounts:
             - name: pod-tmp
@@ -92,11 +102,19 @@ spec:
               mountPath: /tmp/ceph-keyring.sh
               subPath: ceph-keyring.sh
               readOnly: true
+            {{- if $is_internal }}
             - name: ceph-keyring
               mountPath: /tmp/client-keyring
               subPath: key
               readOnly: true
-            {{- if and $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }}
+            {{- end }}
+            {{- if and $is_external $envAll.Values.ceph_client.external_ceph.rbd_user }}
+            - name: external-ceph-keyring
+              mountPath: /tmp/external-ceph-client-keyring
+              subPath: key
+              readOnly: true
+            {{- end }}
+            {{- if and $is_internal $envAll.Values.ceph_client.enable_external_ceph_backend $envAll.Values.ceph_client.external_ceph.rbd_user }}
             - name: external-ceph-keyring
               mountPath: /tmp/external-ceph-client-keyring
               subPath: key
@@ -189,6 +207,7 @@ spec:
             {{- if eq "true" (include "cinder.utils.has_ceph_backend" $envAll) }}
             - name: etcceph
               mountPath: /etc/ceph
+            {{- if eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll) }}
             - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
@@ -197,6 +216,7 @@ spec:
               mountPath: /tmp/client-keyring
               subPath: key
               readOnly: true
+            {{- end }}
             {{- if .Values.ceph_client.enable_external_ceph_backend }}
             {{- if .Values.ceph_client.external_ceph.configmap }}
             - name: external-ceph-etc
@@ -318,25 +338,27 @@ spec:
         {{- if eq "true" (include "cinder.utils.has_ceph_backend" $envAll) }}
         - name: etcceph
           emptyDir: {}
+        {{- if eq "true" (include "cinder.utils.has_internal_ceph_backend" $envAll) }}
         - name: ceph-etc
           configMap:
             name: {{ .Values.ceph_client.configmap }}
             defaultMode: 0444
+        - name: ceph-keyring
+          secret:
+            secretName: {{ .Values.secrets.rbd.volume | quote }}
+        {{- end }}
         {{- if and .Values.ceph_client.enable_external_ceph_backend .Values.ceph_client.external_ceph.configmap }}
         - name: external-ceph-etc
           configMap:
             name: {{ .Values.ceph_client.external_ceph.configmap }}
             defaultMode: 0444
         {{- end }}
-        - name: ceph-keyring
-          secret:
-            secretName: {{ .Values.secrets.rbd.volume | quote }}
         {{- if and .Values.ceph_client.enable_external_ceph_backend .Values.ceph_client.external_ceph.rbd_user }}
         - name: external-ceph-keyring
           secret:
             secretName: {{ .Values.secrets.rbd.volume_external | quote }}
-        {{ end }}
-        {{ end }}
+        {{- end }}
+        {{- end }}
         {{- if eq ( split "://" .Values.conf.cinder.coordination.backend_url )._0 "file" }}
         # NOTE (portdirect): this will need to be set to a shared mount amongst all cinder
         # pods for the coordination backend to be fully functional.

cinder/templates/utils/_has_internal_ceph_backend.tpl

@@ -0,0 +1,26 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "cinder.utils.has_internal_ceph_backend" -}}
+  {{- $has_internal := false -}}
+  {{- range $_, $backend := .Values.conf.backends -}}
+    {{- if kindIs "map" $backend -}}
+      {{- if and (eq $backend.volume_driver "cinder.volume.drivers.rbd.RBDDriver") (ne $backend.rbd_ceph_conf "/etc/ceph/external-ceph.conf") -}}
+        {{- $has_internal = true -}}
+      {{- end -}}
+    {{- end -}}
+  {{- end -}}
+  {{- $has_internal -}}
+{{- end -}}
+

cinder/values.yaml

@@ -850,7 +850,11 @@ conf:
       # via the endpoints section.
       osapi_volume_listen_port: null
       enabled_backends: "rbd1"
+      # When using external Ceph, add the external backend name here, e.g.:
+      # enabled_backends: "rbd1,external_rbd"
       default_volume_type: "rbd1"
+      # When using external Ceph as default, change this to the external backend name, e.g.:
+      # default_volume_type: "external_rbd"
       # NOTE(portdirect): "cinder.backup.drivers.ceph"  and
       # "cinder.backup.drivers.posix" also supported
       # NOTE(rchurch): As of Stein, drivers by class name are required