Add separate headless service for ZooKeeper statefulset

README.md

@@ -418,6 +418,12 @@ helm upgrade -n <namespace> -f values.yaml <pulsar-release-name> apachepulsar/pu
 
 For more detailed information, see our [Upgrading](http://pulsar.apache.org/docs/helm-upgrade/) guide.
 
+## Upgrading to Helm chart version 4.6.0 (upcoming release)
+
+The ZooKeeper StatefulSet has been modified to use a separate headless service and a separate ClusterIP service.
+The StatefulSet will be deleted and re-created during an upgrade. Deleting the StatefulSet will not delete data. The pods will
+remain running until the upgrade has replaced them. The deletion is handled using a Helm pre-upgrade hook, which runs a Kubernetes job using a container that contains `kubectl`. The image is `alpine/k8s` by default and is configurable under the `images.kubectl` key in values.yaml.
+
 ## Upgrading to Helm chart version 4.2.0
 
 ### TLS configuration for ZooKeeper has changed

charts/pulsar/templates/_zookeeper.tpl

@@ -18,12 +18,19 @@ under the License.
 */}}
 
 {{/*
-Define the pulsar zookeeper
+Define the pulsar zookeeper service (ordinary ClusterIP, used by brokers/bookies)
 */}}
 {{- define "pulsar.zookeeper.service" -}}
 {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}
 {{- end }}
 
+{{/*
+Define the pulsar zookeeper headless service (used as the StatefulSet serviceName for pod DNS)
+*/}}
+{{- define "pulsar.zookeeper.service.headless" -}}
+{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-headless
+{{- end }}
+
 {{/*
 Define the pulsar zookeeper
 */}}

charts/pulsar/templates/bookkeeper-cluster-initialize.yaml

@@ -75,7 +75,7 @@ spec:
               echo "user provided zookeepers {{ $zk }} are unreachable... check in 3 seconds ..." && sleep 3;
             done;
             {{ else }}
-            until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}.{{ template "pulsar.namespace" . }}; do
+            until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.zookeeper.service.headless" . }}.{{ template "pulsar.namespace" . }}; do
               sleep 3;
             done;
             {{- end}}

charts/pulsar/templates/pulsar-cluster-initialize.yaml

@@ -83,7 +83,7 @@ spec:
               echo "user provided zookeepers {{ $zk }} are unreachable... check in 3 seconds ..." && sleep 3;
             done;
             {{ else if .Values.components.zookeeper }}
-            until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}.{{ template "pulsar.namespace" . }}; do
+            until nslookup {{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-{{ add (.Values.zookeeper.replicaCount | int) -1 }}.{{ template "pulsar.zookeeper.service.headless" . }}.{{ template "pulsar.namespace" . }}; do
               sleep 3;
             done;
             {{- end }}

charts/pulsar/templates/zookeeper-headless-service.yaml

@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# deploy zookeeper only when `components.zookeeper` is true
+{{- if .Values.components.zookeeper }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: "{{ template "pulsar.zookeeper.service.headless" . }}"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.zookeeper.component }}
+{{- with .Values.zookeeper.headlessService.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+spec:
+  ports:
+    - name: http
+      port: {{ .Values.zookeeper.ports.http }}
+    - name: "{{ .Values.tcpPrefix }}follower"
+      port: {{ .Values.zookeeper.ports.follower }}
+    - name: "{{ .Values.tcpPrefix }}leader-election"
+      port: {{ .Values.zookeeper.ports.leaderElection }}
+    - name: "{{ .Values.tcpPrefix }}client"
+      port: {{ .Values.zookeeper.ports.client }}
+    {{- if .Values.zookeeper.ports.admin }}
+    - name: admin
+      port: {{ .Values.zookeeper.ports.admin }}
+    {{- end }}
+    {{- if and .Values.tls.enabled .Values.tls.zookeeper.enabled }}
+    - name: "{{ .Values.tlsPrefix }}client-tls"
+      port: {{ .Values.zookeeper.ports.clientTls }}
+    {{- end }}
+  clusterIP: None
+  publishNotReadyAddresses: true
+  selector:
+    {{- include "pulsar.matchLabels" . | nindent 4 }}
+    component: {{ .Values.zookeeper.component }}
+{{- end }}

charts/pulsar/templates/zookeeper-service.yaml

@@ -27,7 +27,6 @@ metadata:
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.zookeeper.component }}
-  annotations:
 {{- with .Values.zookeeper.service.annotations }}
   annotations:
 {{ toYaml . | indent 4 }}
@@ -37,10 +36,6 @@ spec:
     # prometheus needs to access /metrics endpoint
     - name: http
       port: {{ .Values.zookeeper.ports.http }}
-    - name: "{{ .Values.tcpPrefix }}follower"
-      port: {{ .Values.zookeeper.ports.follower }}
-    - name: "{{ .Values.tcpPrefix }}leader-election"
-      port: {{ .Values.zookeeper.ports.leaderElection }}
     - name: "{{ .Values.tcpPrefix }}client"
       port: {{ .Values.zookeeper.ports.client }}
     {{- if .Values.zookeeper.ports.admin }}
@@ -51,8 +46,6 @@ spec:
     - name: "{{ .Values.tlsPrefix }}client-tls"
       port: {{ .Values.zookeeper.ports.clientTls }}
     {{- end }}
-  clusterIP: None
-  publishNotReadyAddresses: true
   selector:
     {{- include "pulsar.matchLabels" . | nindent 4 }}
     component: {{ .Values.zookeeper.component }}

charts/pulsar/templates/zookeeper-statefulset-upgrade.yaml

@@ -0,0 +1,115 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# only when `components.zookeeper` is true and `zookeeper.statefulsetUpgrade.enabled` is true,
+# this pre-upgrade hook job will be created to clean up the old zookeeper statefulset if the existing statefulset is created 
+# by a chart older than 4.6.0, which has a different headless service name and will cause issue if not deleted before
+# the new statefulset is created.
+{{- if and .Values.components.zookeeper .Values.zookeeper.statefulsetUpgrade.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-sts-cleanup
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.zookeeper.component }}-sts-cleanup
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-10"
+    "helm.sh/hook-delete-policy": hook-succeeded
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Release.Name }}-sts-cleanup
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.zookeeper.component }}-sts-cleanup
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-10"
+    "helm.sh/hook-delete-policy": hook-succeeded
+rules:
+  - apiGroups: ["apps"]
+    resources: ["statefulsets"]
+    verbs: ["list", "get", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Release.Name }}-sts-cleanup
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.zookeeper.component }}-sts-cleanup
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-10"
+    "helm.sh/hook-delete-policy": hook-succeeded
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}-sts-cleanup
+roleRef:
+  kind: Role
+  name: {{ .Release.Name }}-sts-cleanup
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-sts-cleanup
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.zookeeper.component }}-sts-cleanup
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      serviceAccountName: {{ .Release.Name }}-sts-cleanup
+      restartPolicy: Never
+      containers:
+        - name: sts-cleanup
+          image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.kubectl "root" .) }}"
+          command:
+            - sh
+            - -c
+            - |
+              STS="{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}"
+              CHART_LABEL=$(kubectl get statefulset "$STS" -o jsonpath='{.metadata.labels.chart}' 2>/dev/null || true)
+              if [ -z "$CHART_LABEL" ]; then
+                echo "StatefulSet $STS not found or has no chart label, skipping delete"
+                exit 0
+              fi
+              VERSION="${CHART_LABEL#pulsar-}"
+              MAJOR="$(echo "$VERSION" | cut -d. -f1)"
+              MINOR="$(echo "$VERSION" | cut -d. -f2)"
+              if [ "$MAJOR" -lt 4 ] || { [ "$MAJOR" -eq 4 ] && [ "$MINOR" -lt 6 ]; }; then
+                echo "Chart version $CHART_LABEL is older than pulsar-4.6.0, deleting StatefulSet $STS"
+                kubectl delete statefulset "$STS" --cascade=orphan --ignore-not-found
+              else
+                echo "Chart version $CHART_LABEL is 4.6.0 or newer, skipping delete"
+              fi
+{{- end }}

charts/pulsar/templates/zookeeper-statefulset.yaml

@@ -29,7 +29,7 @@ metadata:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.zookeeper.component }}
 spec:
-  serviceName: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}"
+  serviceName: "{{ template "pulsar.zookeeper.service.headless" . }}"
   replicas: {{ .Values.zookeeper.replicaCount }}
   selector:
     matchLabels:

charts/pulsar/values.yaml

@@ -219,6 +219,9 @@ images:
     repository: oxia/oxia
     tag: 0.15.3
     pullPolicy:
+  kubectl:
+    repository: alpine/k8s
+    tag: 1.32.12
 
 ## TLS
 ## templates/tls-certs.yaml
@@ -572,6 +575,14 @@ zookeeper:
   ##
   service:
     annotations: {}
+  ## Zookeeper headless service
+  ## templates/zookeeper-headless-service.yaml
+  ##
+  headlessService:
+    annotations: {}
+  ## Zookeeper statefulset upgrade job
+  statefulsetUpgrade:
+    enabled: true
   ## Zookeeper PodDisruptionBudget
   ## templates/zookeeper-pdb.yaml
   ##