diff --git a/README.md b/README.md index c32f15a2..7047af11 100644 --- a/README.md +++ b/README.md @@ -418,6 +418,12 @@ helm upgrade -n -f values.yaml apachepulsar/pu For more detailed information, see our [Upgrading](http://pulsar.apache.org/docs/helm-upgrade/) guide. +## Upgrading to Helm chart version 4.6.0 (upcoming release) + +The ZooKeeper StatefulSet has been modified to use a separate headless service and a separate ClusterIP service. +The StatefulSet will be deleted and re-created during an upgrade. Deleting the StatefulSet will not delete data. The pods will +remain running until the upgrade has replaced them. The deletion is handled using a Helm pre-upgrade hook, which runs a Kubernetes job using a container that contains `kubectl`. The image is `alpine/k8s` by default and is configurable under the `images.kubectl` key in values.yaml. + ## Upgrading to Helm chart version 4.2.0 ### TLS configuration for ZooKeeper has changed diff --git a/charts/pulsar/templates/_zookeeper.tpl b/charts/pulsar/templates/_zookeeper.tpl index dc86d40b..b3bdcd9d 100644 --- a/charts/pulsar/templates/_zookeeper.tpl +++ b/charts/pulsar/templates/_zookeeper.tpl @@ -18,12 +18,19 @@ under the License. */}} {{/* -Define the pulsar zookeeper +Define the pulsar zookeeper service (ordinary ClusterIP, used by brokers/bookies) */}} {{- define "pulsar.zookeeper.service" -}} {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }} {{- end }} +{{/* +Define the pulsar zookeeper headless service (used as the StatefulSet serviceName for pod DNS) +*/}} +{{- define "pulsar.zookeeper.service.headless" -}} +{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-headless +{{- end }} + {{/* Define the pulsar zookeeper */}} diff --git a/charts/pulsar/templates/bookkeeper-cluster-initialize.yaml b/charts/pulsar/templates/bookkeeper-cluster-initialize.yaml index 77fbac2c..3269cc66 100755 --- a/charts/pulsar/templates/bookkeeper-cluster-initialize.yaml +++ b/charts/pulsar/templates/bookkeeper-cluster-initialize.yaml @@ -75,7 +75,7 @@ spec: echo "user provided zookeepers {{ $zk }} are unreachable... check in 3 seconds ..." && sleep 3; done; {{ else }} - until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}.{{ template "pulsar.namespace" . }}; do + until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.zookeeper.service.headless" . }}.{{ template "pulsar.namespace" . }}; do sleep 3; done; {{- end}} diff --git a/charts/pulsar/templates/pulsar-cluster-initialize.yaml b/charts/pulsar/templates/pulsar-cluster-initialize.yaml index 158a22d0..3099856a 100755 --- a/charts/pulsar/templates/pulsar-cluster-initialize.yaml +++ b/charts/pulsar/templates/pulsar-cluster-initialize.yaml @@ -83,7 +83,7 @@ spec: echo "user provided zookeepers {{ $zk }} are unreachable... check in 3 seconds ..." && sleep 3; done; {{ else if .Values.components.zookeeper }} - until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}.{{ template "pulsar.namespace" . }}; do + until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.zookeeper.service.headless" . }}.{{ template "pulsar.namespace" . }}; do sleep 3; done; {{- end }} diff --git a/charts/pulsar/templates/zookeeper-headless-service.yaml b/charts/pulsar/templates/zookeeper-headless-service.yaml new file mode 100644 index 00000000..8c99fa69 --- /dev/null +++ b/charts/pulsar/templates/zookeeper-headless-service.yaml @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# deploy zookeeper only when `components.zookeeper` is true +{{- if .Values.components.zookeeper }} +apiVersion: v1 +kind: Service +metadata: + name: "{{ template "pulsar.zookeeper.service.headless" . }}" + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.zookeeper.component }} +{{- with .Values.zookeeper.headlessService.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + ports: + - name: http + port: {{ .Values.zookeeper.ports.http }} + - name: "{{ .Values.tcpPrefix }}follower" + port: {{ .Values.zookeeper.ports.follower }} + - name: "{{ .Values.tcpPrefix }}leader-election" + port: {{ .Values.zookeeper.ports.leaderElection }} + - name: "{{ .Values.tcpPrefix }}client" + port: {{ .Values.zookeeper.ports.client }} + {{- if .Values.zookeeper.ports.admin }} + - name: admin + port: {{ .Values.zookeeper.ports.admin }} + {{- end }} + {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }} + - name: "{{ .Values.tlsPrefix }}client-tls" + port: {{ .Values.zookeeper.ports.clientTls }} + {{- end }} + clusterIP: None + publishNotReadyAddresses: true + selector: + {{- include "pulsar.matchLabels" . | nindent 4 }} + component: {{ .Values.zookeeper.component }} +{{- end }} diff --git a/charts/pulsar/templates/zookeeper-service.yaml b/charts/pulsar/templates/zookeeper-service.yaml index a28f4799..da3a02eb 100644 --- a/charts/pulsar/templates/zookeeper-service.yaml +++ b/charts/pulsar/templates/zookeeper-service.yaml @@ -27,7 +27,6 @@ metadata: labels: {{- include "pulsar.standardLabels" . | nindent 4 }} component: {{ .Values.zookeeper.component }} - annotations: {{- with .Values.zookeeper.service.annotations }} annotations: {{ toYaml . | indent 4 }} @@ -37,10 +36,6 @@ spec: # prometheus needs to access /metrics endpoint - name: http port: {{ .Values.zookeeper.ports.http }} - - name: "{{ .Values.tcpPrefix }}follower" - port: {{ .Values.zookeeper.ports.follower }} - - name: "{{ .Values.tcpPrefix }}leader-election" - port: {{ .Values.zookeeper.ports.leaderElection }} - name: "{{ .Values.tcpPrefix }}client" port: {{ .Values.zookeeper.ports.client }} {{- if .Values.zookeeper.ports.admin }} @@ -51,8 +46,6 @@ spec: - name: "{{ .Values.tlsPrefix }}client-tls" port: {{ .Values.zookeeper.ports.clientTls }} {{- end }} - clusterIP: None - publishNotReadyAddresses: true selector: {{- include "pulsar.matchLabels" . | nindent 4 }} component: {{ .Values.zookeeper.component }} diff --git a/charts/pulsar/templates/zookeeper-statefulset-upgrade.yaml b/charts/pulsar/templates/zookeeper-statefulset-upgrade.yaml new file mode 100644 index 00000000..fa70df10 --- /dev/null +++ b/charts/pulsar/templates/zookeeper-statefulset-upgrade.yaml @@ -0,0 +1,115 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# only when `components.zookeeper` is true and `zookeeper.statefulsetUpgrade.enabled` is true, +# this pre-upgrade hook job will be created to clean up the old zookeeper statefulset if the existing statefulset is created +# by a chart older than 4.6.0, which has a different headless service name and will cause issue if not deleted before +# the new statefulset is created. +{{- if and .Values.components.zookeeper .Values.zookeeper.statefulsetUpgrade.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-sts-cleanup + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.zookeeper.component }}-sts-cleanup + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-sts-cleanup + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.zookeeper.component }}-sts-cleanup + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": hook-succeeded +rules: + - apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["list", "get", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-sts-cleanup + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.zookeeper.component }}-sts-cleanup + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": hook-succeeded +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-sts-cleanup +roleRef: + kind: Role + name: {{ .Release.Name }}-sts-cleanup + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-sts-cleanup + namespace: {{ template "pulsar.namespace" . }} + labels: + {{- include "pulsar.standardLabels" . | nindent 4 }} + component: {{ .Values.zookeeper.component }}-sts-cleanup + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded +spec: + backoffLimit: 1 + template: + spec: + serviceAccountName: {{ .Release.Name }}-sts-cleanup + restartPolicy: Never + containers: + - name: sts-cleanup + image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.kubectl "root" .) }}" + command: + - sh + - -c + - | + STS="{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}" + CHART_LABEL=$(kubectl get statefulset "$STS" -o jsonpath='{.metadata.labels.chart}' 2>/dev/null || true) + if [ -z "$CHART_LABEL" ]; then + echo "StatefulSet $STS not found or has no chart label, skipping delete" + exit 0 + fi + VERSION="${CHART_LABEL#pulsar-}" + MAJOR="$(echo "$VERSION" | cut -d. -f1)" + MINOR="$(echo "$VERSION" | cut -d. -f2)" + if [ "$MAJOR" -lt 4 ] || { [ "$MAJOR" -eq 4 ] && [ "$MINOR" -lt 6 ]; }; then + echo "Chart version $CHART_LABEL is older than pulsar-4.6.0, deleting StatefulSet $STS" + kubectl delete statefulset "$STS" --cascade=orphan --ignore-not-found + else + echo "Chart version $CHART_LABEL is 4.6.0 or newer, skipping delete" + fi +{{- end }} \ No newline at end of file diff --git a/charts/pulsar/templates/zookeeper-statefulset.yaml b/charts/pulsar/templates/zookeeper-statefulset.yaml index 1834f4af..cf962bea 100755 --- a/charts/pulsar/templates/zookeeper-statefulset.yaml +++ b/charts/pulsar/templates/zookeeper-statefulset.yaml @@ -29,7 +29,7 @@ metadata: {{- include "pulsar.standardLabels" . | nindent 4 }} component: {{ .Values.zookeeper.component }} spec: - serviceName: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}" + serviceName: "{{ template "pulsar.zookeeper.service.headless" . }}" replicas: {{ .Values.zookeeper.replicaCount }} selector: matchLabels: diff --git a/charts/pulsar/values.yaml b/charts/pulsar/values.yaml index b1ed8885..9f8620bd 100755 --- a/charts/pulsar/values.yaml +++ b/charts/pulsar/values.yaml @@ -219,6 +219,9 @@ images: repository: oxia/oxia tag: 0.15.3 pullPolicy: + kubectl: + repository: alpine/k8s + tag: 1.32.12 ## TLS ## templates/tls-certs.yaml @@ -572,6 +575,14 @@ zookeeper: ## service: annotations: {} + ## Zookeeper headless service + ## templates/zookeeper-headless-service.yaml + ## + headlessService: + annotations: {} + ## Zookeeper statefulset upgrade job + statefulsetUpgrade: + enabled: true ## Zookeeper PodDisruptionBudget ## templates/zookeeper-pdb.yaml ##