Add separate headless service for brokers with publishNotReadyAddresses

.ci/helm.sh

@@ -25,8 +25,8 @@ PULSAR_CHART_VERSION=${PULSAR_CHART_VERSION:-"local"}
 OUTPUT_BIN=${CHARTS_HOME}/output/bin
 KIND_BIN=$OUTPUT_BIN/kind
 HELM=${OUTPUT_BIN}/helm
-KUBECTL=${OUTPUT_BIN}/kubectl
-NAMESPACE=pulsar
+: ${KUBECTL:=$OUTPUT_BIN/kubectl}
+: ${NAMESPACE:=pulsar}
 CLUSTER=pulsar-ci
 : ${CLUSTER_ID:=$(uuidgen)}
 K8S_LOGS_DIR="${K8S_LOGS_DIR:-/tmp/k8s-logs}"

README.md

@@ -420,10 +420,14 @@ For more detailed information, see our [Upgrading](http://pulsar.apache.org/docs
 
 ## Upgrading to Helm chart version 4.6.0 (upcoming release)
 
-The ZooKeeper StatefulSet has been modified to use a separate headless service and a separate ClusterIP service.
-The StatefulSet will be deleted and re-created during an upgrade. Deleting the StatefulSet will not delete data. The pods will
+The ZooKeeper StatefulSet and Broker StatefulSet have been modified to use a separate headless service and a separate ClusterIP service.
+The StatefulSet will be deleted and re-created during an upgrade. Deleting the StatefulSet will not delete data for ZooKeeper. The pods will
 remain running until the upgrade has replaced them. The deletion is handled using a Helm pre-upgrade hook, which runs a Kubernetes job using a container that contains `kubectl`. The image is `alpine/k8s` by default and is configurable under the `images.kubectl` key in values.yaml.
 
+When TLS is enabled for ZooKeeper and Brokers, it is recommended to perform a rolling restart after the new certificates have been issued by cert-manager. The hostnames of the broker and zookeeper pods have changed and the certificate CRDs reflect this change.
+
+In addition, the default for the main Broker service has been changed from headless to default ClusterIP service.
+
 ## Upgrading to Helm chart version 4.2.0
 
 ### TLS configuration for ZooKeeper has changed

charts/pulsar/templates/_broker.tpl

@@ -18,12 +18,19 @@ under the License.
 */}}
 
 {{/*
-Define the pulsar brroker service
+Define the pulsar broker service (ordinary ClusterIP, used by clients)
 */}}
 {{- define "pulsar.broker.service" -}}
 {{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}
 {{- end }}
 
+{{/*
+Define the pulsar broker headless service (used as the StatefulSet serviceName for pod DNS)
+*/}}
+{{- define "pulsar.broker.service.headless" -}}
+{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-headless
+{{- end }}
+
 {{/*
 Define the hostname
 */}}

charts/pulsar/templates/_certs.tpl

@@ -102,6 +102,9 @@ spec:
 {{- if .tlsConfig.dnsNames }}
 {{ toYaml .tlsConfig.dnsNames | indent 4 }}
 {{- end }}
+    {{- if or (eq .componentConfig.component "broker") (eq .componentConfig.component "zookeeper") }}
+    - {{ printf "*.%s-%s-headless.%s.svc.%s" (include "pulsar.fullname" .root) .componentConfig.component (include "pulsar.namespace" .root) .root.Values.clusterDomain | quote }}
+    {{- end }}
     - {{ printf "*.%s-%s.%s.svc.%s" (include "pulsar.fullname" .root) .componentConfig.component (include "pulsar.namespace" .root) .root.Values.clusterDomain | quote }}
     - {{ printf "%s-%s" (include "pulsar.fullname" .root) .componentConfig.component | quote }}
   # Issuer references are always required.

charts/pulsar/templates/broker-headless-service.yaml

@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# deploy broker only when `components.broker` is true
+{{- if .Values.components.broker }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: "{{ template "pulsar.broker.service.headless" . }}"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.broker.component }}
+{{- with .Values.broker.headlessService.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+spec:
+  ports:
+  - name: http
+    port: {{ .Values.broker.ports.http }}
+  {{- if not (and .Values.tls.enabled .Values.tls.broker.enabled .Values.tls.function_instance.enabled) }}
+  - name: "{{ .Values.tcpPrefix }}pulsar"
+    port: {{ .Values.broker.ports.pulsar }}
+  {{- end }}
+  {{- if and .Values.tls.enabled .Values.tls.broker.enabled }}
+  - name: https
+    port: {{ .Values.broker.ports.https }}
+  - name: "{{ .Values.tlsPrefix }}pulsarssl"
+    port: {{ .Values.broker.ports.pulsarssl }}
+  {{- end }}
+  clusterIP: None
+  publishNotReadyAddresses: true
+  selector:
+    {{- include "pulsar.matchLabels" . | nindent 4 }}
+    component: {{ .Values.broker.component }}
+{{- end }}

charts/pulsar/templates/broker-service.yaml

@@ -36,7 +36,7 @@ spec:
   # prometheus needs to access /metrics endpoint
   - name: http
     port: {{ .Values.broker.ports.http }}
-  {{- if or (not .Values.tls.enabled) (not .Values.tls.broker.enabled) }}
+  {{- if not (and .Values.tls.enabled .Values.tls.broker.enabled .Values.tls.function_instance.enabled) }}
   - name: "{{ .Values.tcpPrefix }}pulsar"
     port: {{ .Values.broker.ports.pulsar }}
   {{- end }}

charts/pulsar/templates/broker-statefulset-upgrade.yaml

@@ -0,0 +1,115 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# only when `components.broker` is true and `broker.statefulsetUpgrade.enabled` is true,
+# this pre-upgrade hook job will be created to clean up the old broker statefulset if the existing statefulset is created 
+# by a chart older than 4.6.0, which has a different headless service name and will cause issue if not deleted before
+# the new statefulset is created.
+{{- if and .Values.components.broker .Values.broker.statefulsetUpgrade.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-sts-cleanup"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.broker.component }}-sts-cleanup
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-10"
+    "helm.sh/hook-delete-policy": hook-succeeded
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-sts-cleanup"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.broker.component }}-sts-cleanup
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-10"
+    "helm.sh/hook-delete-policy": hook-succeeded
+rules:
+  - apiGroups: ["apps"]
+    resources: ["statefulsets"]
+    verbs: ["list", "get", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-sts-cleanup"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.broker.component }}-sts-cleanup
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-10"
+    "helm.sh/hook-delete-policy": hook-succeeded
+subjects:
+  - kind: ServiceAccount
+    name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-sts-cleanup"
+roleRef:
+  kind: Role
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-sts-cleanup"
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-sts-cleanup"
+  namespace: {{ template "pulsar.namespace" . }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: {{ .Values.broker.component }}-sts-cleanup
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}-sts-cleanup"
+      restartPolicy: Never
+      containers:
+        - name: sts-cleanup
+          image: "{{ template "pulsar.imageFullName" (dict "image" .Values.images.kubectl "root" .) }}"
+          command:
+            - sh
+            - -c
+            - |
+              STS="{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
+              CHART_LABEL=$(kubectl get statefulset "$STS" -o jsonpath='{.metadata.labels.chart}' 2>/dev/null || true)
+              if [ -z "$CHART_LABEL" ]; then
+                echo "StatefulSet $STS not found or has no chart label, skipping delete"
+                exit 0
+              fi
+              VERSION="${CHART_LABEL#pulsar-}"
+              MAJOR="$(echo "$VERSION" | cut -d. -f1)"
+              MINOR="$(echo "$VERSION" | cut -d. -f2)"
+              if [ "$MAJOR" -lt 4 ] || { [ "$MAJOR" -eq 4 ] && [ "$MINOR" -lt 6 ]; }; then
+                echo "Chart version $CHART_LABEL is older than pulsar-4.6.0, deleting StatefulSet $STS"
+                kubectl delete statefulset "$STS" --cascade=orphan --ignore-not-found
+              else
+                echo "Chart version $CHART_LABEL is 4.6.0 or newer, skipping delete"
+              fi
+{{- end }}

charts/pulsar/templates/broker-statefulset.yaml

@@ -30,7 +30,7 @@ metadata:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
     component: {{ .Values.broker.component }}
 spec:
-  serviceName: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.component }}"
+  serviceName: "{{ template "pulsar.broker.service.headless" . }}"
   {{- if not .Values.broker.autoscaling.enabled }}
   replicas: {{ .Values.broker.replicaCount }}
   {{- end }}

charts/pulsar/templates/zookeeper-statefulset-upgrade.yaml

@@ -25,7 +25,7 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: {{ .Release.Name }}-sts-cleanup
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-sts-cleanup"
   namespace: {{ template "pulsar.namespace" . }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
@@ -38,7 +38,7 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: {{ .Release.Name }}-sts-cleanup
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-sts-cleanup"
   namespace: {{ template "pulsar.namespace" . }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
@@ -55,7 +55,7 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: {{ .Release.Name }}-sts-cleanup
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-sts-cleanup"
   namespace: {{ template "pulsar.namespace" . }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
@@ -66,16 +66,16 @@ metadata:
     "helm.sh/hook-delete-policy": hook-succeeded
 subjects:
   - kind: ServiceAccount
-    name: {{ .Release.Name }}-sts-cleanup
+    name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-sts-cleanup"
 roleRef:
   kind: Role
-  name: {{ .Release.Name }}-sts-cleanup
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-sts-cleanup"
   apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ .Release.Name }}-sts-cleanup
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-sts-cleanup"
   namespace: {{ template "pulsar.namespace" . }}
   labels:
     {{- include "pulsar.standardLabels" . | nindent 4 }}
@@ -88,7 +88,7 @@ spec:
   backoffLimit: 1
   template:
     spec:
-      serviceAccountName: {{ .Release.Name }}-sts-cleanup
+      serviceAccountName: "{{ template "pulsar.fullname" . }}-{{ .Values.zookeeper.component }}-sts-cleanup"
       restartPolicy: Never
       containers:
         - name: sts-cleanup

charts/pulsar/values.yaml

@@ -1229,8 +1229,17 @@ broker:
     # 2. '' -> non-headless service, k8s picks an IPv4 address
     # 3. 'None' -> headless
     # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-clusterip
-    clusterIP: "None"
+    clusterIP: ""
     annotations: {}
+  ## Broker headless service with publishNotReadyAddresses set to true and used for statefulset's pod hostname resolution
+  ## This service is used for broker pod DNS names in the broker statefulset.
+  ## templates/broker-headless-service.yaml
+  ##
+  headlessService:
+    annotations: {}
+  ## Broker statefulset upgrade job
+  statefulsetUpgrade:
+    enabled: true
   ## Broker PodDisruptionBudget
   ## templates/broker-pdb.yaml
   ##