From a23b0076dfd53060fcd6e1efe933a8b0d20891ae Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Thu, 1 May 2025 16:07:59 +0200 Subject: [PATCH 1/2] [helm] Add TLS support on yugabyte --- .gitignore | 2 +- build/make-certs.py | 2 -- .../helm-charts/dss/templates/_helpers.tpl | 2 +- .../helm-charts/dss/templates/_volumes.tpl | 22 ++++++++++++++---- .../dss/templates/dss-core-service.yaml | 15 +++++++----- .../dss/templates/schema-manager.yaml | 10 ++++---- deploy/services/helm-charts/dss/values.yaml | 23 ++++++++++++++++++- 7 files changed, 57 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index a24e88084..33a9dce5a 100644 --- a/.gitignore +++ b/.gitignore @@ -131,4 +131,4 @@ go .vscode # terraform -.terraform* \ No newline at end of file +.terraform* diff --git a/build/make-certs.py b/build/make-certs.py index 2ea8f1b0e..1f54b8310 100755 --- a/build/make-certs.py +++ b/build/make-certs.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 import argparse -import itertools -import glob import os import shutil import subprocess diff --git a/deploy/services/helm-charts/dss/templates/_helpers.tpl b/deploy/services/helm-charts/dss/templates/_helpers.tpl index f2e569003..31ddbcc5c 100644 --- a/deploy/services/helm-charts/dss/templates/_helpers.tpl +++ b/deploy/services/helm-charts/dss/templates/_helpers.tpl @@ -50,6 +50,6 @@ yugabyte {{ if .cockroachdbEnabled }} - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.datastoreHost}} --port \"{{.datastorePort}}\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}" {{ else }} - - "ysqlsh --host {{.datastoreHost}} --port \"{{.datastorePort}}\" -c \"SELECT datname FROM pg_database where datname = '{{.schemaName}}';\" | grep {{.schemaName}}" + - "ysqlsh --host {{.datastoreHost}} --port \"{{.datastorePort}}\" \"sslmode=require\" -c \"SELECT datname FROM pg_database where datname = '{{.schemaName}}';\" | grep {{.schemaName}}" {{ end }} {{- end -}} diff --git a/deploy/services/helm-charts/dss/templates/_volumes.tpl b/deploy/services/helm-charts/dss/templates/_volumes.tpl index e21dc30e4..981e5d80c 100644 --- a/deploy/services/helm-charts/dss/templates/_volumes.tpl +++ b/deploy/services/helm-charts/dss/templates/_volumes.tpl @@ -2,27 +2,41 @@ - name: ca-certs secret: defaultMode: 256 - secretName: cockroachdb.ca.crt + secretName: {{ if .cockroachdbEnabled }}cockroachdb.ca.crt{{ else }}yugabyte-tls-client-cert{{ end }} {{- end -}} {{- define "ca-certs:volumeMount" -}} +{{ if .cockroachdbEnabled }} - mountPath: /cockroach/cockroach-certs/ca.crt name: ca-certs subPath: ca.crt +{{ else }} +- mountPath: /opt/yugabyte-certs/ca.crt + name: ca-certs + subPath: root.crt +{{- end -}} {{- end -}} - {{- define "client-certs:volume" -}} - name: client-certs secret: defaultMode: 256 - secretName: cockroachdb.client.root + secretName: {{ if .cockroachdbEnabled }}cockroachdb.client.root{{ else }}yugabyte-tls-client-cert{{ end }} {{- end -}} {{- define "client-certs:volumeMount" -}} +{{ if .cockroachdbEnabled }} - mountPath: /cockroach/cockroach-certs/client.root.crt name: client-certs subPath: client.root.crt - mountPath: /cockroach/cockroach-certs/client.root.key name: client-certs subPath: client.root.key +{{ else }} +- mountPath: /opt/yugabyte-certs/client.yugabyte.crt + name: client-certs + subPath: yugabytedb.crt +- mountPath: /opt/yugabyte-certs/client.yugabyte.key + name: client-certs + subPath: yugabytedb.key +{{- end -}} {{- end -}} @@ -35,4 +49,4 @@ {{- define "public-certs:volumeMount" -}} - mountPath: /public-certs name: public-certs -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index d129031ed..d5c8a1913 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -51,6 +51,9 @@ spec: - --cockroach_ssl_dir=/cockroach/cockroach-certs - --cockroach_ssl_mode=verify-full - --locality={{ .Values.cockroachdb.conf.locality }} +{{ else }} + - --cockroach_ssl_dir=/opt/yugabyte-certs + - --cockroach_ssl_mode=verify-full {{ end }} - --dump_requests=true - --enable_scd={{$dss.enableScd | default true}} @@ -80,12 +83,12 @@ spec: stdin: false tty: false volumeMounts: - {{- include "ca-certs:volumeMount" . | nindent 12 }} - {{- include "client-certs:volumeMount" . | nindent 12 }} - {{- include "public-certs:volumeMount" . | nindent 12 }} + {{- include "ca-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "client-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "public-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} imagePullSecrets: [] terminationGracePeriodSeconds: 30 volumes: - {{- include "ca-certs:volume" . | nindent 8 }} - {{- include "client-certs:volume" . | nindent 8 }} - {{- include "public-certs:volume" . | nindent 8 }} + {{- include "ca-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "client-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "public-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} diff --git a/deploy/services/helm-charts/dss/templates/schema-manager.yaml b/deploy/services/helm-charts/dss/templates/schema-manager.yaml index fbd9b22a6..f5a232d76 100644 --- a/deploy/services/helm-charts/dss/templates/schema-manager.yaml +++ b/deploy/services/helm-charts/dss/templates/schema-manager.yaml @@ -46,6 +46,8 @@ spec: - --schemas_dir=/db-schemas/{{$service}} {{ else }} - --schemas_dir=/db-schemas/yugabyte/{{$service}} + - --cockroach_ssl_dir=/opt/yugabyte-certs/ + - --cockroach_ssl_mode=verify-full {{ end }} - --db_version={{$schemaVersion}} command: @@ -57,12 +59,12 @@ spec: stdin: false tty: false volumeMounts: - {{- include "ca-certs:volumeMount" . | nindent 12 }} - {{- include "client-certs:volumeMount" . | nindent 12 }} + {{- include "ca-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "client-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} imagePullSecrets: [] restartPolicy: OnFailure terminationGracePeriodSeconds: 30 volumes: - {{- include "ca-certs:volume" . | nindent 8 }} - {{- include "client-certs:volume" . | nindent 8 }} + {{- include "ca-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "client-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} {{- end -}} diff --git a/deploy/services/helm-charts/dss/values.yaml b/deploy/services/helm-charts/dss/values.yaml index dfb2ed0e2..b2285f34a 100644 --- a/deploy/services/helm-charts/dss/values.yaml +++ b/deploy/services/helm-charts/dss/values.yaml @@ -16,7 +16,28 @@ cockroachdb: yugabyte: enabled: false + # See https://github.com/yugabyte/charts/blob/master/stable/yugabyte/values.yaml + Image: repository: yugabytedb/yugabyte - # See https://github.com/yugabyte/charts/blob/master/stable/yugabyte/values.yaml + isMultiAz: true + + gflags: + master: + placement_cloud: "cloud-1" + placement_region: "uss-1" + placement_zone: "zone-1" + tserver: + placement_cloud: "cloud-1" + placement_region: "uss-1" + placement_zone: "zone-1" + + masterAddresses: "yb-master-0.yb-masters.default.svc.cluster.local:7100,yb-master-1.yb-masters.default.svc.cluster.local:7100,yb-master-2.yb-masters.default.svc.cluster.local:7100" + + tls: + enabled: true + nodeToNode: true + clientToServer: true + insecure: false + provided: true From 4f12df76376fad380d04119e3b0e026711e120c8 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 3 Jun 2025 10:21:20 +0200 Subject: [PATCH 2/2] Defaults & values --- .../helm-charts/dss/values.example.yaml | 16 ++++- .../helm-charts/dss/values.schema.json | 66 ++++++++++++++++++- deploy/services/helm-charts/dss/values.yaml | 13 +--- 3 files changed, 80 insertions(+), 15 deletions(-) diff --git a/deploy/services/helm-charts/dss/values.example.yaml b/deploy/services/helm-charts/dss/values.example.yaml index 1395cb7bb..1e75f35f9 100644 --- a/deploy/services/helm-charts/dss/values.example.yaml +++ b/deploy/services/helm-charts/dss/values.example.yaml @@ -40,9 +40,19 @@ cockroachdb: yugabyte: enabled: false # See https://github.com/yugabyte/charts/blob/master/stable/yugabyte/values.yaml - Image: - tag: 2.25.1.0-b381 - nameOverride: dss-yugabyte + + gflags: + master: + placement_cloud: "cloud-1" + placement_region: "uss-1" + placement_zone: "zone-1" + tserver: + placement_cloud: "cloud-1" + placement_region: "uss-1" + placement_zone: "zone-1" + + # If you join a yugabyte cluster, set the masterAddresses there + # masterAddresses: "yb-master-0.yb-masters.default.svc.cluster.local:7100,yb-master-1.yb-masters.default.svc.cluster.local:7100,yb-master-2.yb-masters.default.svc.cluster.local:7100" loadBalancers: cockroachdbNodes: diff --git a/deploy/services/helm-charts/dss/values.schema.json b/deploy/services/helm-charts/dss/values.schema.json index 4f8e8b66c..8af475293 100644 --- a/deploy/services/helm-charts/dss/values.schema.json +++ b/deploy/services/helm-charts/dss/values.schema.json @@ -94,11 +94,73 @@ "enabled": { "type": "boolean", "description": "Enable YugabyteDB. YugabyteDB xor CockroachDB should be enabled (only one at at time, at least one)." + }, + "gflags": { + "type": "object", + "properties": { + "master": { + "type": "object", + "properties": { + "placement_cloud": { + "type": "string", + "description": "Name of the cloud where instances are deployed, in a recommended deployment should be set to 'dss'" + }, + "placement_region": { + "type": "string", + "description": "Name of the region where instances are deployed, in a recommended deployment should be set to your organization's name" + }, + "placement_zone": { + "type": "string", + "description": "Name of the zone where instances are deployed, in a recommended deployment should be set to 'zone'" + } + }, + "required": [ + "placement_cloud", + "placement_region", + "placement_zone" + ], + "additionalItems": true + }, + "tserver": { + "type": "object", + "properties": { + "placement_cloud": { + "type": "string", + "description": "Name of the cloud where instances are deployed, in a recommended deployment should be set to 'dss'" + }, + "placement_region": { + "type": "string", + "description": "Name of the region where instances are deployed, in a recommended deployment should be set to your organization's name" + }, + "placement_zone": { + "type": "string", + "description": "Name of the zone where instances are deployed, in a recommended deployment should be set to 'zone'" + } + }, + "required": [ + "placement_cloud", + "placement_region", + "placement_zone" + ], + "additionalItems": true + } + }, + "required": [ + "master", + "tserver" + ], + "additionalItems": true + }, + "masterAddresses": { + "type": "string", + "description": "A comma-separated list of master addresses to use. For a local cluster, you can omit it to use the default values, if you join an existing cluster you need to set it the list of public addresses of all master yugabyte nodes" } }, "required": [ - "enabled" - ] + "enabled", + "gflags" + ], + "additionalItems": true }, "loadBalancers": { "description": "Load balancers configuration", diff --git a/deploy/services/helm-charts/dss/values.yaml b/deploy/services/helm-charts/dss/values.yaml index b2285f34a..83e053cc6 100644 --- a/deploy/services/helm-charts/dss/values.yaml +++ b/deploy/services/helm-charts/dss/values.yaml @@ -20,18 +20,11 @@ yugabyte: Image: repository: yugabytedb/yugabyte + tag: 2.25.1.0-b381 - isMultiAz: true + nameOverride: dss-yugabyte - gflags: - master: - placement_cloud: "cloud-1" - placement_region: "uss-1" - placement_zone: "zone-1" - tserver: - placement_cloud: "cloud-1" - placement_region: "uss-1" - placement_zone: "zone-1" + isMultiAz: true masterAddresses: "yb-master-0.yb-masters.default.svc.cluster.local:7100,yb-master-1.yb-masters.default.svc.cluster.local:7100,yb-master-2.yb-masters.default.svc.cluster.local:7100"