diff --git a/k8s_helm_charts/README.md b/k8s_helm_charts/README.md index b603cb69..1e227c72 100644 --- a/k8s_helm_charts/README.md +++ b/k8s_helm_charts/README.md @@ -19,19 +19,31 @@ Usage: ```helm -n zdmproxy uninstall zdm-proxy``` - ```helm -n zdmproxy install --set proxy.resources.requests.cpu=1000m --set proxy.resources.requests.memory=2000Mi --set proxy.resources.limits.cpu=1000m --set proxy.resources.limits.memory=2000Mi --set cdm.resources.requests.cpu=1000m --set cdm.resources.requests.memory=2000Mi --set cdm.resources.limits.cpu=1000m --set cdm.resources.limits.memory=2000Mi zdm-proxy ./zdm``` + ```helm -n zdmproxy install --set resources.requests.cpu=1000m --set resources.requests.memory=2000Mi --set resources.limits.cpu=1000m --set resources.limits.memory=2000Mi zdm-proxy ./zdm``` 4. Verify that all components are up and running. - ```kubectl -n zdmproxy get svc,cm,secret,deploy,po -o wide --show-labels``` + ```kubectl -n zdmproxy get svc,ep,po,secret -o wide --show-labels``` + + You can also run ```kubectl -n zdmproxy logs pod/zdm-proxy-0``` to see if there are the following entries in the log, which means everything is working as expected: - You can also run ```kubectl -n zdmproxy logs pod/zdm-proxy-0-xxxxxxx``` to see if there are the following entries in the log, which means everything is working as expected: ``` time="2022-12-14T21:19:57Z" level=info msg="Proxy connected and ready to accept queries on 172.25.132.116:9042" time="2022-12-14T21:19:57Z" level=info msg="Proxy started. Waiting for SIGINT/SIGTERM to shutdown." ``` -5. When you're done, run helm uninstall to remove all objects. +5. Switch primary cluster to target (all proxy pods will automatically roll-restart after the change). + + ```helm -n zdmproxy upgrade zdm-proxy ./zdm --set primaryCluster=TARGET``` + +6. Scale out/in to different number of proxy pods. + + ```helm -n zdmproxy upgrade zdm-proxy ./zdm --set count=5``` + + Note: if you've already switched primary cluster to target, make sure you add ```--set primaryCluster=TARGET``` in this command line as well. An alternative is to directly edit zdm/values.yaml then run helm upgrade. + +7. When you're done, run helm uninstall to remove all objects. + ```helm -n zdmproxy uninstall zdm-proxy``` diff --git a/k8s_helm_charts/zdm/Chart.yaml b/k8s_helm_charts/zdm/Chart.yaml index c7a49506..d33f7188 100644 --- a/k8s_helm_charts/zdm/Chart.yaml +++ b/k8s_helm_charts/zdm/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 1.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "2.0.x" +appVersion: "2.1.x" diff --git a/k8s_helm_charts/zdm/templates/cdm.yaml b/k8s_helm_charts/zdm/templates/cdm.yaml deleted file mode 100644 index 81f972e7..00000000 --- a/k8s_helm_charts/zdm/templates/cdm.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: cdm - name: cdm - namespace: {{ .Values.namespace }} -spec: - replicas: 1 - selector: - matchLabels: - app: cdm - template: - metadata: - labels: - app: cdm - spec: - containers: - - name: cdm - image: {{ .Values.cdm.image.repository }}:{{ .Values.cdm.image.tag }} - resources: - requests: - memory: {{ .Values.cdm.resources.requests.memory }} - cpu: {{ .Values.cdm.resources.requests.cpu }} - limits: - memory: {{ .Values.cdm.resources.limits.memory }} - cpu: {{ .Values.cdm.resources.limits.cpu }} - envFrom: - - configMapRef: - name: {{ include "zdm.fullname" . }} - env: - - name: ZDM_TARGET_SECURE_CONNECT_BUNDLE_PATH - value: /tmp/scb/target.zip - - name: ZDM_ORIGIN_CONTACT_POINTS - valueFrom: - secretKeyRef: - name: zdmproxy - key: origin_contact_points - - name: ZDM_ORIGIN_PORT - valueFrom: - secretKeyRef: - name: zdmproxy - key: origin_port - - name: ZDM_ORIGIN_USERNAME - valueFrom: - secretKeyRef: - name: zdmproxy - key: origin_username - - name: ZDM_ORIGIN_PASSWORD - valueFrom: - secretKeyRef: - name: zdmproxy - key: origin_password - - name: ZDM_TARGET_USERNAME - valueFrom: - secretKeyRef: - name: zdmproxy - key: target_username - - name: ZDM_TARGET_PASSWORD - valueFrom: - secretKeyRef: - name: zdmproxy - key: target_password - volumeMounts: - - name: scb - mountPath: "/tmp/scb" - readOnly: true - volumes: - - name: scb - secret: - secretName: zdmproxy-scb - items: - - key: secure-connect-target.zip - path: target.zip diff --git a/k8s_helm_charts/zdm/templates/configmap.yaml b/k8s_helm_charts/zdm/templates/configmap.yaml deleted file mode 100644 index 086c6c35..00000000 --- a/k8s_helm_charts/zdm/templates/configmap.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "zdm.fullname" . }} - namespace: {{ .Values.namespace }} -data: - ZDM_PRIMARY_CLUSTER: {{ .Values.proxy.primaryCluster | quote }} - ZDM_READ_MODE: {{ .Values.proxy.readMode | quote }} - ZDM_LOG_LEVEL: {{ .Values.proxy.logLevel | quote }} - ZDM_PROXY_MAX_CLIENT_CONNECTIONS: {{ .Values.proxy.maxClientConnections | quote }} - ZDM_METRICS_ENABLED: {{ .Values.proxy.metricsEnabled | quote }} diff --git a/k8s_helm_charts/zdm/templates/service.yaml b/k8s_helm_charts/zdm/templates/service.yaml index f13de8b1..1ef002e4 100644 --- a/k8s_helm_charts/zdm/templates/service.yaml +++ b/k8s_helm_charts/zdm/templates/service.yaml @@ -2,7 +2,7 @@ {{ $zdm_fullname := include "zdm.fullname" . -}} {{- $zdm_labels := include "zdm.labels" . -}} {{- $zdm_selectorLabels := include "zdm.selectorLabels" . -}} -{{- range $index := until (.Values.proxy.count | int) -}} +{{- range $index := until (.Values.count | int) -}} apiVersion: v1 kind: Service metadata: @@ -13,14 +13,14 @@ metadata: app: {{ $zdm_fullname }}-{{ $index }} role: zdmproxy spec: - type: {{ $.Values.service.type }} + type: {{ $.Values.serviceType }} ports: - - port: {{ $.Values.service.port | int }} + - port: {{ $.Values.servicePort | int }} targetPort: 9042 protocol: TCP name: cql selector: {{- $zdm_selectorLabels | nindent 4 }} - app: {{ $zdm_fullname }}-{{ $index }} + statefulset.kubernetes.io/pod-name: {{ $zdm_fullname }}-{{ $index }} --- {{- end -}} diff --git a/k8s_helm_charts/zdm/templates/deployment.yaml b/k8s_helm_charts/zdm/templates/sts.yaml similarity index 62% rename from k8s_helm_charts/zdm/templates/deployment.yaml rename to k8s_helm_charts/zdm/templates/sts.yaml index a00f4e77..50682698 100644 --- a/k8s_helm_charts/zdm/templates/deployment.yaml +++ b/k8s_helm_charts/zdm/templates/sts.yaml @@ -5,46 +5,71 @@ # calculate a variable that contains all proxy service addresses {{ $service_addresses := "" -}} -{{- range $index := until (.Values.proxy.count | int) -}} +{{- range $index := until (.Values.count | int) -}} {{- $service_addresses = printf "%s,$(%s_%s_SERVICE_HOST)" $service_addresses ($zdm_fullname | upper | replace "-" "_") ($index | toString) -}} {{- end -}} {{- $service_addresses = $service_addresses | trimPrefix "," -}} -{{- range $index := until (.Values.proxy.count | int) -}} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: labels: {{- $zdm_labels | nindent 4 }} - app: {{ $zdm_fullname }}-{{ $index }} - name: {{ $zdm_fullname }}-{{ $index }} + app: {{ $zdm_fullname }} + name: {{ $zdm_fullname }} namespace: {{ $.Values.namespace }} spec: - replicas: 1 + serviceName: {{ $zdm_fullname }} + replicas: {{ $.Values.count | int }} selector: matchLabels: {{- $zdm_selectorLabels | nindent 6 }} - app: {{ $zdm_fullname }}-{{ $index }} + app: {{ $zdm_fullname }} template: metadata: labels: {{- $zdm_selectorLabels | nindent 8 }} - app: {{ $zdm_fullname }}-{{ $index }} + app: {{ $zdm_fullname }} spec: containers: - - image: "{{ $.Values.proxy.image.repository }}:{{ $.Values.proxy.image.tag | default $.Chart.AppVersion }}" - name: {{ $zdm_fullname }}-{{ $index }} + - image: "{{ $.Values.imageRepository }}:{{ $.Values.imageTag | default $.Chart.AppVersion }}" + name: {{ $zdm_fullname }} + command: + - sh + - "-c" + - "ZDM_PROXY_TOPOLOGY_INDEX=`echo ${HOSTNAME##*-}` /main" + livenessProbe: + httpGet: + path: /health/liveness + port: httpproxy + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /health/readiness + port: httpproxy + initialDelaySeconds: 10 + periodSeconds: 10 + ports: + - name: httpproxy + containerPort: 14001 + protocol: TCP + - name: cql + containerPort: 9042 + protocol: TCP resources: - requests: - memory: {{ $.Values.proxy.resources.requests.memory | quote }} - cpu: {{ $.Values.proxy.resources.requests.cpu | quote }} - limits: - memory: {{ $.Values.proxy.resources.limits.memory | quote }} - cpu: {{ $.Values.proxy.resources.limits.cpu | quote }} - envFrom: - - configMapRef: - name: {{ $zdm_fullname }} + {{- toYaml .Values.resources | nindent 10 }} env: + - name: ZDM_PRIMARY_CLUSTER + value: {{ $.Values.primaryCluster | upper | quote }} + - name: ZDM_READ_MODE + value: {{ $.Values.readMode | upper | quote }} + - name: ZDM_LOG_LEVEL + value: {{ $.Values.logLevel | upper | quote }} + - name: ZDM_PROXY_MAX_CLIENT_CONNECTIONS + value: {{ $.Values.maxClientConnections | quote }} + - name: ZDM_METRICS_ENABLED + value: {{ $.Values.metricsEnabled | quote }} - name: ZDM_PROXY_LISTEN_ADDRESS valueFrom: fieldRef: @@ -57,8 +82,6 @@ spec: fieldPath: status.podIP - name: ZDM_METRICS_PORT value: "14001" - - name: ZDM_PROXY_TOPOLOGY_INDEX - value: {{ $index | quote }} - name: ZDM_PROXY_TOPOLOGY_ADDRESSES value: {{ $service_addresses }} - name: ZDM_TARGET_SECURE_CONNECT_BUNDLE_PATH @@ -93,8 +116,6 @@ spec: secretKeyRef: name: zdmproxy key: target_password - ports: - - containerPort: 9042 volumeMounts: - name: scb mountPath: "/tmp/scb" @@ -106,5 +127,3 @@ spec: items: - key: secure-connect-target.zip path: target.zip ---- -{{- end -}} diff --git a/k8s_helm_charts/zdm/values.yaml b/k8s_helm_charts/zdm/values.yaml index ded8d0ef..7f21aa2a 100644 --- a/k8s_helm_charts/zdm/values.yaml +++ b/k8s_helm_charts/zdm/values.yaml @@ -2,45 +2,47 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +# namespace to deploy zdm objects in namespace: zdmproxy -proxy: - count: 3 - logLevel: "INFO" - # two options: PRIMARY_ONLY and ASYNC_READS_ON_SECONDARY - readMode: "PRIMARY_ONLY" - # two options: ORIGIN and TARGET - primaryCluster: "ORIGIN" - maxClientConnections: "1000" - metricsEnabled: "true" - resources: - limits: - cpu: 4000m - memory: 2048Mi - requests: - cpu: 4000m - memory: 2048Mi - image: - repository: datastax/zdm-proxy - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -cdm: - resources: - limits: - cpu: 16000m - memory: 32768Mi - requests: - cpu: 16000m - memory: 32768Mi - image: - repository: datastax/cassandra-data-migrator - tag: 2.10.3 +# count is the number of zdm proxy pods (and corresponding services) +count: 3 + +# logLevel is logging level for zdm proxy, available options are: PANIC, FATAL, ERROR, WARN or WARNING, INFO, DEBUG and TRACE +logLevel: "INFO" + +# readMode has two options: PRIMARY_ONLY and ASYNC_READS_ON_SECONDARY +readMode: "PRIMARY_ONLY" + +# primaryCluster has two options: ORIGIN and TARGET +primaryCluster: "ORIGIN" + +# maxClientConnections is the limit on maximum number of client connections per proxy +maxClientConnections: "1000" + +# metricsEnabled is whether to enable metrics/http port on proxy +metricsEnabled: "true" + +# resources request and limit for zdm proxy pods +resources: + limits: + cpu: 4000m + memory: 2048Mi + requests: + cpu: 4000m + memory: 2048Mi + +# imageRepository is the dockerhub repository to pull zdm-proxy image from +imageRepository: datastax/zdm-proxy + +# imageTag defines the tag on dockerhub repo for pulling zdm-proxy image, it overrides the image tag whose default is the chart appVersion. +imageTag: "" + +# serviceType is the type of service used for zdm-proxy pod's associated CQL service +serviceType: ClusterIP + +# servicePort is the CQL port exposed to the rest of the k8s cluster from the zdm-proxy service +servicePort: 9942 nameOverride: "" fullnameOverride: "" - -service: - type: ClusterIP - port: 9942