From 35022920b84320b30a3d3696f02a6d6f3062c60e Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 1 Apr 2025 08:40:51 +0200 Subject: [PATCH 01/18] Helm charts: add gitignore --- deploy/services/helm-charts/dss/.gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 deploy/services/helm-charts/dss/.gitignore diff --git a/deploy/services/helm-charts/dss/.gitignore b/deploy/services/helm-charts/dss/.gitignore new file mode 100644 index 000000000..a4f7ff37c --- /dev/null +++ b/deploy/services/helm-charts/dss/.gitignore @@ -0,0 +1,3 @@ +Chart.lock +charts +values.dev.yaml From 42a27b40b81897c6804959528fa3ebf7fc42790e Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 1 Apr 2025 08:40:09 +0200 Subject: [PATCH 02/18] Add minikube instructions and update helm charts to work with minikube --- deploy/infrastructure/README.md | 11 +++- .../infrastructure/local/minikube/README.md | 51 +++++++++++++++++++ .../dss/templates/_networking-minikuke.tpl | 14 +++++ .../dss/templates/dss-core-service.yaml | 2 +- .../dss/templates/dss-ingress-minikube.yaml | 37 ++++++++++++++ .../helm-charts/dss/values.example.yaml | 4 +- .../helm-charts/dss/values.schema.json | 2 +- 7 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 deploy/infrastructure/local/minikube/README.md create mode 100644 deploy/services/helm-charts/dss/templates/_networking-minikuke.tpl create mode 100644 deploy/services/helm-charts/dss/templates/dss-ingress-minikube.yaml diff --git a/deploy/infrastructure/README.md b/deploy/infrastructure/README.md index d04cda999..964bf512a 100644 --- a/deploy/infrastructure/README.md +++ b/deploy/infrastructure/README.md @@ -9,13 +9,20 @@ The [modules](modules) directory contains the terraform public modules required - [terraform-aws-dss](./modules/terraform-aws-dss/README.md): Amazon Web Services deployment - [terraform-google-dss](./modules/terraform-google-dss/README.md): Google Cloud Engine deployment + ## Dependencies -The [dependencies](dependencies) directory contains submodules used by the public modules described above. They are not expected to be +The [dependencies](dependencies) directory contains submodules used by the public modules described above. They are not expected to be used directly by users. Those submodules are the combination of the cloud specific dependencies `terraform-*-kubernetes` -and `terraform-common-dss`. `terraform-common-dss` module aggregates and outputs the infrastructure configuration +and `terraform-common-dss`. `terraform-common-dss` module aggregates and outputs the infrastructure configuration which can be used as input to the `Services` deployment as shown in the diagram below. ![Infrastructure Modules](../../assets/generated/deploy_infrastructure_modules.png) +## Local + +The [local](local) directory contains various documentation that can be used to spawn a cluster locally. + +- [minikuke](./local/minikube/README.md): Minikube local deployment + ## Utils This [utils folder](utils) contains scripts to help manage the terraform modules and dependencies. See the README in that folder for details. diff --git a/deploy/infrastructure/local/minikube/README.md b/deploy/infrastructure/local/minikube/README.md new file mode 100644 index 000000000..f25234395 --- /dev/null +++ b/deploy/infrastructure/local/minikube/README.md @@ -0,0 +1,51 @@ +# minikube + +This module provide instructions to prepare a local minikube cluster. + +Minikube is going to take care of most of the work by spawning a local kubernetes cluster. + +## Getting started + +### Prerequisites + +Download & install the following tools to your workstation: + +1. Install [minikube](https://minikube.sigs.k8s.io/docs/start/) (First step only). +2. Install tools from [Prerequisites](../../../../build/README.md) + +### Create a new minikube cluster + +1. Run `minikube start -p dss-local-cluster` to create a new cluster. +2. Run `minikube tunnel -p dss-local-cluster` and keep it running to expose LoadBalancer services. + +If needed, you can change the name of the cluster (`dss-local-cluster` in this documentation) as needed. You may also deploy multiple cluster at the same time, using different names. + +### Access to the cluster + +Minikube provide a UI, should you want to keep track of deployment and/or inspect the cluster. To start it, use the following command: + +1. `minikube dashboard -p dss-local-cluster` + +You can also use any other tool as needed. You can switch to the cluster's context by using the following command: + +1. `kubectl config use-context dss-local-cluster` + +### Upload or update local image + +Should you want to run the local docker image that you [built](../../../../build/README.md), run the following commands to upload / update your image + +1. `minikube image -p dss-local-cluster push interuss-local/dss` + +In the helm charts, use `docker.io/interuss-local/dss:latest` as image and be sure to set the `imagePullPolicy` to `Never`. + +## Deployment of the DSS services + +You can now deploy the DSS services using [helm charts](../../../services/helm-charts/dss/README.md). + +Use the `global.cloudProvider` setting with the value `minikube` and deploy the charts on the `dss-local-cluster` kubernetes context. + +To access the service, find the external IP using the `kubectl get services dss-dss-gateway` command. The port 80, without HTTPs is used. + +## Clean up + +To delete all resources, run `minikube delete -p dss-local-cluster`. Note that this operation can't be reverted and all data will be lost. diff --git a/deploy/services/helm-charts/dss/templates/_networking-minikuke.tpl b/deploy/services/helm-charts/dss/templates/_networking-minikuke.tpl new file mode 100644 index 000000000..8034b2c0e --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/_networking-minikuke.tpl @@ -0,0 +1,14 @@ +{{- define "minikube-lb-default-annotations" -}} +{{- end -}} + +{{- define "minikube-lb-crdb-annotations" -}} +{{- end -}} + +{{- define "minikube-lb-spec" -}} +{{- end -}} + +{{- define "minikube-ingress-dss-gateway-annotations" -}} +{{- end -}} + +{{- define "minikube-ingress-spec" -}} +{{- end -}} diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 2a32e3f59..09d90cdfe 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -52,7 +52,7 @@ spec: - --jwks_key_ids={{ $dss.conf.jwksKeyIds | join "," }} {{- end }} - --locality={{ .Values.cockroachdb.conf.locality }} - {{- if $dss.conf.pubKeys}} + {{- if $dss.conf.pubKeys }} - --public_key_files={{ $dss.conf.pubKeys | join "," }} {{- end }} command: diff --git a/deploy/services/helm-charts/dss/templates/dss-ingress-minikube.yaml b/deploy/services/helm-charts/dss/templates/dss-ingress-minikube.yaml new file mode 100644 index 000000000..8bf51ddca --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/dss-ingress-minikube.yaml @@ -0,0 +1,37 @@ +{{- $cloudProvider := $.Values.global.cloudProvider}} +{{- if eq $cloudProvider "minikube" }} +{{/* +Minikube application load balancer Ingress do not support elastic ip assignment yet. Therefore, the +ingress is replaced by a network load balancer (Kubernetes Service of type Load Balancer) +*/}} +{{- with $.Values.loadBalancers.dssGateway }} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + {{- include (printf "%s-ingress-dss-gateway-annotations" $cloudProvider) + (merge . + (dict + "name" "dss-gateway-external" + "cloudProvider" $cloudProvider + ) + ) | nindent 4 + }} + labels: + app: {{$.Release.Name}}-core-service + name: {{$.Release.Name}}-dss-gateway + name: {{$.Release.Name}}-dss-gateway + namespace: default +spec: + {{- include (printf "%s-ingress-spec" $cloudProvider) . | nindent 2 }} + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: {{$.Release.Name}}-core-service + type: LoadBalancer +{{- end }} +{{- end }} diff --git a/deploy/services/helm-charts/dss/values.example.yaml b/deploy/services/helm-charts/dss/values.example.yaml index 16e477bec..e21e0f556 100644 --- a/deploy/services/helm-charts/dss/values.example.yaml +++ b/deploy/services/helm-charts/dss/values.example.yaml @@ -2,6 +2,8 @@ dss: image: docker.io/interuss/dss:v0.15.0 # See https://hub.docker.com/r/interuss/dss/tags for official image releases. + # When running local images in minikube, uncomment the following line + # imagePullPolicy: Never conf: pubKeys: - /test-certs/auth2.pem @@ -23,7 +25,7 @@ cockroachdb: statefulset: replicas: 3 # Must match the number of .loadBalancers.cockroachdbNodes items. - args: + args: # Remove this whole block for minikube - --locality-advertise-addr=zone=interuss-example-google-ew1@$(hostname -f) - --advertise-addr=${HOSTNAME##*-}.db.example.com updateStrategy: diff --git a/deploy/services/helm-charts/dss/values.schema.json b/deploy/services/helm-charts/dss/values.schema.json index 8126f298a..776b98536 100644 --- a/deploy/services/helm-charts/dss/values.schema.json +++ b/deploy/services/helm-charts/dss/values.schema.json @@ -175,7 +175,7 @@ "type": "object", "properties": { "cloudProvider": { - "description": "Cloud provider identifier. `aws` or `google`", + "description": "Cloud provider identifier. `aws`, `google` or `minikube`", "type": "string" } }, From 02ff9c8e3d91b2fd11b0c8de34b59a8d866b8e30 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 1 Apr 2025 08:37:38 +0200 Subject: [PATCH 03/18] Helm charts: add configurable imagePullPolicy --- deploy/services/helm-charts/dss/templates/dss-core-service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 09d90cdfe..b101eef9e 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -58,7 +58,7 @@ spec: command: - core-service image: {{ $dss.image }} - imagePullPolicy: Always + imagePullPolicy: {{ $dss.imagePullPolicy | default "Always" }} name: core-service ports: - containerPort: 8080 From bf0f05b0528ef619898bd0d783f0018cefbec25e Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 1 Apr 2025 17:40:00 +0200 Subject: [PATCH 04/18] Yugabyte: helm charts --- deploy/services/helm-charts/dss/Chart.yaml | 5 +++ .../helm-charts/dss/templates/_helpers.tpl | 41 ++++++++++++++++--- .../templates/cockroachdb-loadbalancers.yaml | 3 ++ .../dss/templates/dss-core-service.yaml | 30 +++++++++----- .../dss/templates/schema-manager.yaml | 29 +++++++++---- .../helm-charts/dss/values.example.yaml | 8 ++++ .../helm-charts/dss/values.schema.json | 26 +++++++++--- deploy/services/helm-charts/dss/values.yaml | 8 ++++ 8 files changed, 121 insertions(+), 29 deletions(-) diff --git a/deploy/services/helm-charts/dss/Chart.yaml b/deploy/services/helm-charts/dss/Chart.yaml index e3043c1d2..cdcee988e 100644 --- a/deploy/services/helm-charts/dss/Chart.yaml +++ b/deploy/services/helm-charts/dss/Chart.yaml @@ -8,3 +8,8 @@ dependencies: - name: cockroachdb repository: https://charts.cockroachdb.com/ version: 10.0.7 + condition: cockroachdb.enabled + - name: yugabyte + repository: https://charts.yugabyte.com/ + version: 2.25.1 + condition: yugabyte.enabled diff --git a/deploy/services/helm-charts/dss/templates/_helpers.tpl b/deploy/services/helm-charts/dss/templates/_helpers.tpl index f7005f370..67670520f 100644 --- a/deploy/services/helm-charts/dss/templates/_helpers.tpl +++ b/deploy/services/helm-charts/dss/templates/_helpers.tpl @@ -1,9 +1,34 @@ -{{- define "cockroachImage" -}} +{{- define "databaseImage" -}} +{{- if $.Values.cockroachdb.enabled -}} {{ (printf "%s:%s" $.Values.cockroachdb.image.repository $.Values.cockroachdb.image.tag) }} +{{- else -}} +{{ (printf "%s:%s" $.Values.yugabyte.Image.repository $.Values.yugabyte.Image.tag) }} {{- end -}} +{{- end -}} + +{{- define "databasePort" -}} +{{- if $.Values.cockroachdb.enabled -}} +26257 +{{- else -}} +5433 +{{- end -}} +{{- end -}} + +{{- define "databaseUser" -}} +{{- if $.Values.cockroachdb.enabled -}} +root +{{- else -}} +yugabyte +{{- end -}} +{{- end -}} + -{{- define "cockroachHost" -}} +{{- define "databaseHost" -}} +{{- if $.Values.cockroachdb.enabled -}} {{- printf "%s-public.default" $.Values.cockroachdb.fullnameOverride -}} +{{- else -}} +{{- printf "yb-tservers.default" -}} +{{- end -}} {{- end -}} {{- define "init-container-wait-for-http" -}} @@ -13,14 +38,18 @@ {{- end -}} {{- define "init-container-wait-for-schema" -}} -{{/*For some reason, calling the template cockroachImage fails here.*/}} +{{/*For some reason, calling the template databaseImage fails here.*/}} - name: wait-for-schema-{{.schemaName}} - image: {{.cockroachImage}} + image: {{.databaseImage}} volumeMounts: {{- include "ca-certs:volumeMount" . | nindent 4 }} {{- include "client-certs:volumeMount" . | nindent 4 }} command: - sh - -c - - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.cockroachHost}} --port \"26257\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}" -{{- end -}} \ No newline at end of file +{{ if .cockroachdb }} + - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.databaseHost}} --port \"{{.databasePort}}\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}" +{{ else }} + - "ysqlsh --host {{.databaseHost}} --port \"{{.databasePort}}\" -c \"SELECT datname FROM pg_database where datname = '{{.schemaName}}';\" | grep {{.schemaName}}" +{{ end }} +{{- end -}} diff --git a/deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml b/deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml index 19e002bca..211ef1f6e 100644 --- a/deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml +++ b/deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml @@ -1,5 +1,7 @@ {{- $cloudProvider := $.Values.global.cloudProvider}} +{{- if $.Values.cockroachdb.enabled }} + # Node Gateways {{- range $i, $lb := .Values.loadBalancers.cockroachdbNodes }} --- @@ -32,3 +34,4 @@ spec: statefulset.kubernetes.io/pod-name: {{$.Release.Name}}-cockroachdb-{{$i}} type: LoadBalancer {{- end }} +{{- end }} diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index b101eef9e..31f84c05e 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -1,9 +1,17 @@ {{- $dss := .Values.dss -}} -{{- $cockroachImage := (include "cockroachImage" .) -}} -{{- $cockroachHost := (include "cockroachHost" .) -}} -{{- $waitForCockroachDB := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $cockroachHost)) -}} -{{- $waitForRIDSchema := include "init-container-wait-for-schema" (dict "schemaName" "rid" "cockroachImage" $cockroachImage "cockroachHost" $cockroachHost) -}} -{{- $waitForSCDSchema := include "init-container-wait-for-schema" (dict "schemaName" "scd" "cockroachImage" $cockroachImage "cockroachHost" $cockroachHost) -}} + +{{- $databaseImage := (include "databaseImage" .) -}} +{{- $databaseHost := (include "databaseHost" .) -}} +{{- $databasePort := (include "databasePort" .) -}} +{{- $databaseUser := (include "databaseUser" .) -}} + +{{- $waitForDatabase := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $databaseHost)) -}} +{{- if .Values.yugabyte.enabled }} +{{- $waitForDatabase = include "init-container-wait-for-http" (dict "serviceName" "yb-tserver" "url" (printf "http://%s:9000/status" $databaseHost)) -}} +{{- end -}} + +{{- $waitForRIDSchema := include "init-container-wait-for-schema" (dict "schemaName" "rid" "databaseImage" $databaseImage "databasePort" $databasePort "databaseHost" $databaseHost "cockroachdb" .Values.cockroachdb.enabled ) -}} +{{- $waitForSCDSchema := include "init-container-wait-for-schema" (dict "schemaName" "scd" "databaseImage" $databaseImage "databasePort" $databasePort "databaseHost" $databaseHost "cockroachdb" .Values.cockroachdb.enabled ) -}} --- apiVersion: apps/v1 @@ -29,18 +37,21 @@ spec: app: {{.Release.Name}}-core-service spec: initContainers: - {{- $waitForCockroachDB | nindent 8 }} + {{- $waitForDatabase | nindent 8 }} {{- $waitForRIDSchema | nindent 8 }} {{- $waitForSCDSchema | nindent 8 }} containers: - args: - --accepted_jwt_audiences={{$dss.conf.hostname}} - --addr=:8080 - - --cockroach_host={{$cockroachHost}} - - --cockroach_port=26257 + - --cockroach_host={{ $databaseHost }} + - --cockroach_port={{ $databasePort }} + - --cockroach_user={{ $databaseUser }} +{{ if $.Values.cockroachdb.enabled }} - --cockroach_ssl_dir=/cockroach/cockroach-certs - --cockroach_ssl_mode=verify-full - - --cockroach_user=root + - --locality={{ .Values.cockroachdb.conf.locality }} +{{ end }} - --dump_requests=true - --enable_scd={{$dss.enableScd | default true}} - --garbage_collector_spec=@every 30m @@ -51,7 +62,6 @@ spec: {{- if $dss.conf.jwksKeyIds }} - --jwks_key_ids={{ $dss.conf.jwksKeyIds | join "," }} {{- end }} - - --locality={{ .Values.cockroachdb.conf.locality }} {{- if $dss.conf.pubKeys }} - --public_key_files={{ $dss.conf.pubKeys | join "," }} {{- end }} diff --git a/deploy/services/helm-charts/dss/templates/schema-manager.yaml b/deploy/services/helm-charts/dss/templates/schema-manager.yaml index 137795f9b..00459a989 100644 --- a/deploy/services/helm-charts/dss/templates/schema-manager.yaml +++ b/deploy/services/helm-charts/dss/templates/schema-manager.yaml @@ -1,9 +1,18 @@ {{- $image := .Values.dss.image }} -{{- $cockroachHost := (include "cockroachHost" .) -}} +{{- $databaseHost := (include "databaseHost" .) -}} +{{- $databasePort := (include "databasePort" .) -}} +{{- $databaseUser := (include "databaseUser" .) -}} {{- $jobVersion := .Release.Revision -}} {{/* Jobs template definition is immutable, using the revision in the name forces the job to be recreated at each helm upgrade. */}} -{{- $waitForCockroachDB := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $cockroachHost)) -}} -{{- range $service, $schemaVersion := dict "rid" "4.0.0" "scd" "3.2.0" }} +{{- $waitForDatabase := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $databaseHost)) -}} +{{- $schemas := dict "rid" "4.0.0" "scd" "3.2.0" }} + +{{- if .Values.yugabyte.enabled }} +{{- $waitForDatabase = include "init-container-wait-for-http" (dict "serviceName" "yb-tserver" "url" (printf "http://%s:9000/status" $databaseHost)) -}} +{{- $schemas = dict "rid" "1.0.1" "scd" "1.0.1" }} +{{- end -}} + +{{- range $service, $schemaVersion := $schemas }} --- apiVersion: batch/v1 kind: Job @@ -24,17 +33,21 @@ spec: name: {{$service}}-schema-manager-{{$jobVersion}} spec: initContainers: - {{- $waitForCockroachDB | nindent 8 }} + {{- $waitForDatabase | nindent 8 }} containers: - args: - migrate - - --cockroach_host={{$cockroachHost}} - - --cockroach_port=26257 + - --cockroach_host={{ $databaseHost }} + - --cockroach_port={{ $databasePort }} + - --cockroach_user={{ $databaseUser }} +{{ if $.Values.cockroachdb.enabled }} - --cockroach_ssl_dir=/cockroach/cockroach-certs - --cockroach_ssl_mode=verify-full - - --cockroach_user=root - - --db_version={{$schemaVersion}} - --schemas_dir=/db-schemas/{{$service}} +{{ else }} + - --schemas_dir=/db-schemas/yugabyte/{{$service}} +{{ end }} + - --db_version={{$schemaVersion}} command: - db-manager - migrate diff --git a/deploy/services/helm-charts/dss/values.example.yaml b/deploy/services/helm-charts/dss/values.example.yaml index e21e0f556..da1321c8e 100644 --- a/deploy/services/helm-charts/dss/values.example.yaml +++ b/deploy/services/helm-charts/dss/values.example.yaml @@ -13,6 +13,7 @@ dss: enableScd: true cockroachdb: + enabled: false # See https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/values.yaml image: tag: v24.1.3 @@ -36,6 +37,13 @@ cockroachdb: persistentVolume: storageClass: standard +yugabyte: + enabled: true + # See https://github.com/yugabyte/charts/blob/master/stable/yugabyte/values.yaml + Image: + tag: 2.25.1.0-b381 + nameOverride: dss-yugabyte + loadBalancers: cockroachdbNodes: - ip: 192.168.0.21 diff --git a/deploy/services/helm-charts/dss/values.schema.json b/deploy/services/helm-charts/dss/values.schema.json index 776b98536..4f8e8b66c 100644 --- a/deploy/services/helm-charts/dss/values.schema.json +++ b/deploy/services/helm-charts/dss/values.schema.json @@ -5,6 +5,10 @@ "description": "Cockroach DB related configuration", "type": "object", "properties": { + "enabled": { + "type": "boolean", + "description": "Enable CockroachDB. YugabyteDB xor CockroachDB should be enabled (only one at at time, at least one)." + }, "image": { "type": "object", "properties": { @@ -59,10 +63,7 @@ }, "additionalItems": true } - }, - "required": [ - "args" - ] + } }, "storage": { "type": "object", @@ -82,7 +83,21 @@ "image", "fullnameOverride", "conf", - "statefulset" + "statefulset", + "enabled" + ] + }, + "yugabyte": { + "description": "Yugabyte related configuration", + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable YugabyteDB. YugabyteDB xor CockroachDB should be enabled (only one at at time, at least one)." + } + }, + "required": [ + "enabled" ] }, "loadBalancers": { @@ -184,6 +199,7 @@ }, "required": [ "cockroachdb", + "yugabyte", "loadBalancers", "dss", "global" diff --git a/deploy/services/helm-charts/dss/values.yaml b/deploy/services/helm-charts/dss/values.yaml index f869add13..dfb2ed0e2 100644 --- a/deploy/services/helm-charts/dss/values.yaml +++ b/deploy/services/helm-charts/dss/values.yaml @@ -1,5 +1,6 @@ # Default configuration cockroachdb: + enabled: true # See https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/values.yaml image: repository: cockroachdb/cockroach @@ -12,3 +13,10 @@ cockroachdb: enabled: false ingress: enabled: false + +yugabyte: + enabled: false + Image: + repository: yugabytedb/yugabyte + + # See https://github.com/yugabyte/charts/blob/master/stable/yugabyte/values.yaml From 5209853cd61689b232e8eefe64ac83509d63d5bd Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 1 Apr 2025 17:40:00 +0200 Subject: [PATCH 05/18] Yugabyte: helm charts --- deploy/services/helm-charts/dss/Chart.yaml | 5 +++ .../helm-charts/dss/templates/_helpers.tpl | 41 ++++++++++++++++--- .../templates/cockroachdb-loadbalancers.yaml | 3 ++ .../dss/templates/dss-core-service.yaml | 31 +++++++++----- .../dss/templates/schema-manager.yaml | 29 +++++++++---- .../helm-charts/dss/values.example.yaml | 8 ++++ .../helm-charts/dss/values.schema.json | 26 +++++++++--- deploy/services/helm-charts/dss/values.yaml | 8 ++++ 8 files changed, 122 insertions(+), 29 deletions(-) diff --git a/deploy/services/helm-charts/dss/Chart.yaml b/deploy/services/helm-charts/dss/Chart.yaml index e3043c1d2..cdcee988e 100644 --- a/deploy/services/helm-charts/dss/Chart.yaml +++ b/deploy/services/helm-charts/dss/Chart.yaml @@ -8,3 +8,8 @@ dependencies: - name: cockroachdb repository: https://charts.cockroachdb.com/ version: 10.0.7 + condition: cockroachdb.enabled + - name: yugabyte + repository: https://charts.yugabyte.com/ + version: 2.25.1 + condition: yugabyte.enabled diff --git a/deploy/services/helm-charts/dss/templates/_helpers.tpl b/deploy/services/helm-charts/dss/templates/_helpers.tpl index f7005f370..67670520f 100644 --- a/deploy/services/helm-charts/dss/templates/_helpers.tpl +++ b/deploy/services/helm-charts/dss/templates/_helpers.tpl @@ -1,9 +1,34 @@ -{{- define "cockroachImage" -}} +{{- define "databaseImage" -}} +{{- if $.Values.cockroachdb.enabled -}} {{ (printf "%s:%s" $.Values.cockroachdb.image.repository $.Values.cockroachdb.image.tag) }} +{{- else -}} +{{ (printf "%s:%s" $.Values.yugabyte.Image.repository $.Values.yugabyte.Image.tag) }} {{- end -}} +{{- end -}} + +{{- define "databasePort" -}} +{{- if $.Values.cockroachdb.enabled -}} +26257 +{{- else -}} +5433 +{{- end -}} +{{- end -}} + +{{- define "databaseUser" -}} +{{- if $.Values.cockroachdb.enabled -}} +root +{{- else -}} +yugabyte +{{- end -}} +{{- end -}} + -{{- define "cockroachHost" -}} +{{- define "databaseHost" -}} +{{- if $.Values.cockroachdb.enabled -}} {{- printf "%s-public.default" $.Values.cockroachdb.fullnameOverride -}} +{{- else -}} +{{- printf "yb-tservers.default" -}} +{{- end -}} {{- end -}} {{- define "init-container-wait-for-http" -}} @@ -13,14 +38,18 @@ {{- end -}} {{- define "init-container-wait-for-schema" -}} -{{/*For some reason, calling the template cockroachImage fails here.*/}} +{{/*For some reason, calling the template databaseImage fails here.*/}} - name: wait-for-schema-{{.schemaName}} - image: {{.cockroachImage}} + image: {{.databaseImage}} volumeMounts: {{- include "ca-certs:volumeMount" . | nindent 4 }} {{- include "client-certs:volumeMount" . | nindent 4 }} command: - sh - -c - - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.cockroachHost}} --port \"26257\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}" -{{- end -}} \ No newline at end of file +{{ if .cockroachdb }} + - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.databaseHost}} --port \"{{.databasePort}}\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}" +{{ else }} + - "ysqlsh --host {{.databaseHost}} --port \"{{.databasePort}}\" -c \"SELECT datname FROM pg_database where datname = '{{.schemaName}}';\" | grep {{.schemaName}}" +{{ end }} +{{- end -}} diff --git a/deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml b/deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml index 19e002bca..211ef1f6e 100644 --- a/deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml +++ b/deploy/services/helm-charts/dss/templates/cockroachdb-loadbalancers.yaml @@ -1,5 +1,7 @@ {{- $cloudProvider := $.Values.global.cloudProvider}} +{{- if $.Values.cockroachdb.enabled }} + # Node Gateways {{- range $i, $lb := .Values.loadBalancers.cockroachdbNodes }} --- @@ -32,3 +34,4 @@ spec: statefulset.kubernetes.io/pod-name: {{$.Release.Name}}-cockroachdb-{{$i}} type: LoadBalancer {{- end }} +{{- end }} diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 2a32e3f59..424d59045 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -1,9 +1,17 @@ {{- $dss := .Values.dss -}} -{{- $cockroachImage := (include "cockroachImage" .) -}} -{{- $cockroachHost := (include "cockroachHost" .) -}} -{{- $waitForCockroachDB := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $cockroachHost)) -}} -{{- $waitForRIDSchema := include "init-container-wait-for-schema" (dict "schemaName" "rid" "cockroachImage" $cockroachImage "cockroachHost" $cockroachHost) -}} -{{- $waitForSCDSchema := include "init-container-wait-for-schema" (dict "schemaName" "scd" "cockroachImage" $cockroachImage "cockroachHost" $cockroachHost) -}} + +{{- $databaseImage := (include "databaseImage" .) -}} +{{- $databaseHost := (include "databaseHost" .) -}} +{{- $databasePort := (include "databasePort" .) -}} +{{- $databaseUser := (include "databaseUser" .) -}} + +{{- $waitForDatabase := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $databaseHost)) -}} +{{- if .Values.yugabyte.enabled }} +{{- $waitForDatabase = include "init-container-wait-for-http" (dict "serviceName" "yb-tserver" "url" (printf "http://%s:9000/status" $databaseHost)) -}} +{{- end -}} + +{{- $waitForRIDSchema := include "init-container-wait-for-schema" (dict "schemaName" "rid" "databaseImage" $databaseImage "databasePort" $databasePort "databaseHost" $databaseHost "cockroachdb" .Values.cockroachdb.enabled ) -}} +{{- $waitForSCDSchema := include "init-container-wait-for-schema" (dict "schemaName" "scd" "databaseImage" $databaseImage "databasePort" $databasePort "databaseHost" $databaseHost "cockroachdb" .Values.cockroachdb.enabled ) -}} --- apiVersion: apps/v1 @@ -29,18 +37,21 @@ spec: app: {{.Release.Name}}-core-service spec: initContainers: - {{- $waitForCockroachDB | nindent 8 }} + {{- $waitForDatabase | nindent 8 }} {{- $waitForRIDSchema | nindent 8 }} {{- $waitForSCDSchema | nindent 8 }} containers: - args: - --accepted_jwt_audiences={{$dss.conf.hostname}} - --addr=:8080 - - --cockroach_host={{$cockroachHost}} - - --cockroach_port=26257 + - --cockroach_host={{ $databaseHost }} + - --cockroach_port={{ $databasePort }} + - --cockroach_user={{ $databaseUser }} +{{ if $.Values.cockroachdb.enabled }} - --cockroach_ssl_dir=/cockroach/cockroach-certs - --cockroach_ssl_mode=verify-full - - --cockroach_user=root + - --locality={{ .Values.cockroachdb.conf.locality }} +{{ end }} - --dump_requests=true - --enable_scd={{$dss.enableScd | default true}} - --garbage_collector_spec=@every 30m @@ -52,7 +63,7 @@ spec: - --jwks_key_ids={{ $dss.conf.jwksKeyIds | join "," }} {{- end }} - --locality={{ .Values.cockroachdb.conf.locality }} - {{- if $dss.conf.pubKeys}} + {{- if $dss.conf.pubKeys }} - --public_key_files={{ $dss.conf.pubKeys | join "," }} {{- end }} command: diff --git a/deploy/services/helm-charts/dss/templates/schema-manager.yaml b/deploy/services/helm-charts/dss/templates/schema-manager.yaml index 137795f9b..00459a989 100644 --- a/deploy/services/helm-charts/dss/templates/schema-manager.yaml +++ b/deploy/services/helm-charts/dss/templates/schema-manager.yaml @@ -1,9 +1,18 @@ {{- $image := .Values.dss.image }} -{{- $cockroachHost := (include "cockroachHost" .) -}} +{{- $databaseHost := (include "databaseHost" .) -}} +{{- $databasePort := (include "databasePort" .) -}} +{{- $databaseUser := (include "databaseUser" .) -}} {{- $jobVersion := .Release.Revision -}} {{/* Jobs template definition is immutable, using the revision in the name forces the job to be recreated at each helm upgrade. */}} -{{- $waitForCockroachDB := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $cockroachHost)) -}} -{{- range $service, $schemaVersion := dict "rid" "4.0.0" "scd" "3.2.0" }} +{{- $waitForDatabase := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $databaseHost)) -}} +{{- $schemas := dict "rid" "4.0.0" "scd" "3.2.0" }} + +{{- if .Values.yugabyte.enabled }} +{{- $waitForDatabase = include "init-container-wait-for-http" (dict "serviceName" "yb-tserver" "url" (printf "http://%s:9000/status" $databaseHost)) -}} +{{- $schemas = dict "rid" "1.0.1" "scd" "1.0.1" }} +{{- end -}} + +{{- range $service, $schemaVersion := $schemas }} --- apiVersion: batch/v1 kind: Job @@ -24,17 +33,21 @@ spec: name: {{$service}}-schema-manager-{{$jobVersion}} spec: initContainers: - {{- $waitForCockroachDB | nindent 8 }} + {{- $waitForDatabase | nindent 8 }} containers: - args: - migrate - - --cockroach_host={{$cockroachHost}} - - --cockroach_port=26257 + - --cockroach_host={{ $databaseHost }} + - --cockroach_port={{ $databasePort }} + - --cockroach_user={{ $databaseUser }} +{{ if $.Values.cockroachdb.enabled }} - --cockroach_ssl_dir=/cockroach/cockroach-certs - --cockroach_ssl_mode=verify-full - - --cockroach_user=root - - --db_version={{$schemaVersion}} - --schemas_dir=/db-schemas/{{$service}} +{{ else }} + - --schemas_dir=/db-schemas/yugabyte/{{$service}} +{{ end }} + - --db_version={{$schemaVersion}} command: - db-manager - migrate diff --git a/deploy/services/helm-charts/dss/values.example.yaml b/deploy/services/helm-charts/dss/values.example.yaml index 16e477bec..3d2d14e03 100644 --- a/deploy/services/helm-charts/dss/values.example.yaml +++ b/deploy/services/helm-charts/dss/values.example.yaml @@ -11,6 +11,7 @@ dss: enableScd: true cockroachdb: + enabled: false # See https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/values.yaml image: tag: v24.1.3 @@ -34,6 +35,13 @@ cockroachdb: persistentVolume: storageClass: standard +yugabyte: + enabled: true + # See https://github.com/yugabyte/charts/blob/master/stable/yugabyte/values.yaml + Image: + tag: 2.25.1.0-b381 + nameOverride: dss-yugabyte + loadBalancers: cockroachdbNodes: - ip: 192.168.0.21 diff --git a/deploy/services/helm-charts/dss/values.schema.json b/deploy/services/helm-charts/dss/values.schema.json index 8126f298a..697b1d3e3 100644 --- a/deploy/services/helm-charts/dss/values.schema.json +++ b/deploy/services/helm-charts/dss/values.schema.json @@ -5,6 +5,10 @@ "description": "Cockroach DB related configuration", "type": "object", "properties": { + "enabled": { + "type": "boolean", + "description": "Enable CockroachDB. YugabyteDB xor CockroachDB should be enabled (only one at at time, at least one)." + }, "image": { "type": "object", "properties": { @@ -59,10 +63,7 @@ }, "additionalItems": true } - }, - "required": [ - "args" - ] + } }, "storage": { "type": "object", @@ -82,7 +83,21 @@ "image", "fullnameOverride", "conf", - "statefulset" + "statefulset", + "enabled" + ] + }, + "yugabyte": { + "description": "Yugabyte related configuration", + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable YugabyteDB. YugabyteDB xor CockroachDB should be enabled (only one at at time, at least one)." + } + }, + "required": [ + "enabled" ] }, "loadBalancers": { @@ -184,6 +199,7 @@ }, "required": [ "cockroachdb", + "yugabyte", "loadBalancers", "dss", "global" diff --git a/deploy/services/helm-charts/dss/values.yaml b/deploy/services/helm-charts/dss/values.yaml index f869add13..dfb2ed0e2 100644 --- a/deploy/services/helm-charts/dss/values.yaml +++ b/deploy/services/helm-charts/dss/values.yaml @@ -1,5 +1,6 @@ # Default configuration cockroachdb: + enabled: true # See https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/values.yaml image: repository: cockroachdb/cockroach @@ -12,3 +13,10 @@ cockroachdb: enabled: false ingress: enabled: false + +yugabyte: + enabled: false + Image: + repository: yugabytedb/yugabyte + + # See https://github.com/yugabyte/charts/blob/master/stable/yugabyte/values.yaml From 471c6fc63a4562857e170b42b41589c647dc2113 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Thu, 1 May 2025 16:07:59 +0200 Subject: [PATCH 06/18] [helm] Add TLS support on yugabyte --- .gitignore | 3 +- build/apply-certs-yugabyte.sh | 54 +++ build/make-certs-yugabyte.py | 364 ++++++++++++++++++ build/make-certs.py | 2 - .../helm-charts/dss/templates/_helpers.tpl | 2 +- .../helm-charts/dss/templates/_volumes.tpl | 22 +- .../dss/templates/dss-core-service.yaml | 15 +- .../dss/templates/schema-manager.yaml | 10 +- deploy/services/helm-charts/dss/values.yaml | 23 ++ 9 files changed, 477 insertions(+), 18 deletions(-) create mode 100755 build/apply-certs-yugabyte.sh create mode 100755 build/make-certs-yugabyte.py diff --git a/.gitignore b/.gitignore index a24e88084..689976da1 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ test_result # Generated configs build/generated/ build/workspace/ +build/workspace-yugabyte/ build/cockroachdb.yaml build/values.yaml build/dss.yaml @@ -131,4 +132,4 @@ go .vscode # terraform -.terraform* \ No newline at end of file +.terraform* diff --git a/build/apply-certs-yugabyte.sh b/build/apply-certs-yugabyte.sh new file mode 100755 index 000000000..8bcb72cb4 --- /dev/null +++ b/build/apply-certs-yugabyte.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [ "$1" == "" ]; then + echo "usage: ./apply-certs.sh " + exit 1 +fi + +if [ "$2" == "" ]; then + echo "usage: ./apply-certs.sh " + exit 1 +fi + +echo '=========================================================================' +echo '= Note that errors below are acceptable as long as the terminal message =' +echo '= is success. =' +echo '=========================================================================' + +set -e +set -x + +# Paths to directories in which to store certificates and generated YAML files. +CONTEXT="$1" +DIR="$(pwd)" +NAMESPACE="$2" + +# Replace characters breaking folder names +WORKSPACE=$(echo "${CONTEXT}" | tr ':/' '_') +CLIENTS_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/client_certs_dir" +MASTER_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/master_certs_dir" +TSERVER_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/tserver_certs_dir" +# CA_KEY_DIR="$DIR/workspace/$WORKSPACE/ca_key_dir" +# CA_CRT_DIR="$DIR/workspace/$WORKSPACE/ca_certs_dir" +JWT_PUBLIC_CERTS_DIR="$DIR/jwt-public-certs" + +# Delete previous secrets in case they have changed. +kubectl create namespace "$NAMESPACE" --context "$CONTEXT" || true + +kubectl delete secret yb-master-yugabyte-tls-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true +kubectl delete secret yb-tserver-yugabyte-tls-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true +kubectl delete secret yugabyte-tls-client-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true +kubectl delete secret dss.public.certs --namespace "$NAMESPACE" --context "$CONTEXT" || true + +kubectl create secret generic yb-master-yugabyte-tls-cert --namespace "$NAMESPACE" --from-file "$MASTER_CERTS_DIR" --context "$CONTEXT" +kubectl create secret generic yb-tserver-yugabyte-tls-cert --namespace "$NAMESPACE" --from-file "$TSERVER_CERTS_DIR" --context "$CONTEXT" +kubectl create secret generic yugabyte-tls-client-cert --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" + + +kubectl create secret generic dss.public.certs --namespace "$NAMESPACE" --from-file "$JWT_PUBLIC_CERTS_DIR" --context "$CONTEXT" + +echo '=========================================================================' +echo '= Secrets uploaded successfully. =' +echo '=========================================================================' diff --git a/build/make-certs-yugabyte.py b/build/make-certs-yugabyte.py new file mode 100755 index 000000000..3f209999e --- /dev/null +++ b/build/make-certs-yugabyte.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 + +import argparse +import os +import shutil +import subprocess + + +class YugabyteCluster(object): + + def __init__(self, cluster_context, namespace, ca_cert_to_join=None): + self._ca_cert_to_join = ca_cert_to_join + self._cluster_context = cluster_context + self._namespace = namespace + + @property + def ca_cert_to_join(self): + return self._ca_cert_to_join + + @property + def namespace(self): + return self._namespace + + @property + def directory(self): + # Replace characters breaking folder names + def remove_special_chars(s: str): + for c in [":", "/"]: + s = s.replace(c, "_") + return s + + return os.path.join( + os.getcwd(), + "workspace-yugabyte", remove_special_chars(self._cluster_context) + ) + + @property + def ca_key_dir(self): + return os.path.join(self.directory, "ca_key_dir") + + @property + def ca_key_file(self): + return os.path.join(self.ca_key_dir, "ca.key") + + @property + def ca_cert_file(self): + return os.path.join(self.ca_key_dir, "ca.crt") + + @property + def ca_conf(self): + return os.path.join(self.ca_key_dir, "ca.conf") + + @property + def ca_certs_dir(self): + return os.path.join(self.directory, "ca_certs_dir") + + @property + def client_certs_dir(self): + return os.path.join(self.directory, "client_certs_dir") + + @property + def master_certs_dir(self): + return os.path.join(self.directory, "master_certs_dir") + + @property + def tserver_certs_dir(self): + return os.path.join(self.directory, "tserver_certs_dir") + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Creates certificates for a new Cockroachdb cluster" + ) + parser.add_argument( + "--cluster-context", + metavar="CLUSTER_CONTEXT", + required=True, + help="kubernetes cluster context name", + ) + parser.add_argument( + "--namespace", + metavar="NAMESPACE", + required=True, + help="kubernetes cluster namespace you are deploying to.", + ) + # TODO + # parser.add_argument( + # "--node-address", + # metavar="ADDRESS", + # nargs="*", + # default=[], + # help="extra addresses to add to the node certificate", + # ) + parser.add_argument( + "--ca-cert-to-join", + metavar="FILENAME", + help="file containing an existing CA cert of a cluster to join.", + ) + parser.add_argument( + "--overwrite-ca-cert", + action="store_true", + default=False, + help="True to generate new CA certs, false to use the existing one", + ) + parser.add_argument( + "--node-count", + metavar="NODE_COUNT", + default="3", + help="Number of yugabyte nodes in the cluster, default to 3", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + cr = YugabyteCluster(args.cluster_context, args.namespace, args.ca_cert_to_join) + + # Create the generated directories. + if not os.path.exists("workspace-yugabyte"): + os.makedirs("workspace-yugabyte") + + if not os.path.exists(cr.directory): + os.makedirs(cr.directory) + + create_ca = not os.path.exists(cr.ca_key_dir) or args.overwrite_ca_cert + if create_ca: + # Create a new CA. + # Delete and recreate the ca_certs_dir. + shutil.rmtree(cr.ca_key_dir, ignore_errors=True) + os.mkdir(cr.ca_key_dir) + + # Build master, tserver and client certs. + os.mkdir(cr.master_certs_dir) + os.mkdir(cr.tserver_certs_dir) + os.mkdir(cr.client_certs_dir) + + if create_ca: + + with open(cr.ca_conf, "w") as f: + f.write( + f""" + [ ca ] + default_ca = my_ca + +[ my_ca ] +default_days = 3650 + +serial = {cr.ca_key_dir}/serial.txt +database = {cr.ca_key_dir}/index.txt +default_md = sha256 +policy = my_policy + +[ my_policy ] + +organizationName = supplied +commonName = supplied + +[req] +prompt=no +distinguished_name = my_distinguished_name +x509_extensions = my_extensions + +[ my_distinguished_name ] +organizationName = Yugabyte +commonName = CA for YugabyteDB + +[ my_extensions ] +keyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment,keyCertSign +basicConstraints = critical,CA:true,pathlen:1 + +""" + ) + + with open(f"{cr.ca_key_dir}/serial.txt", "w") as f: + f.write("01") + + with open(f"{cr.ca_key_dir}/index.txt", "w") as f: + f.write("") + + subprocess.check_call(["openssl", "genrsa", "-out", cr.ca_key_file]) + + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-x509", + "-days", + "3650", + "-config", + cr.ca_conf, + "-key", + cr.ca_key_file, + "-out", + cr.ca_cert_file, + ] + ) + + ### CLIENT CERTIFICATE + + # Copy CA + shutil.copy(cr.ca_cert_file, f"{cr.client_certs_dir}/root.crt") + + for client in ["yugabytedb"]: + + crt_name = f"{cr.client_certs_dir}/{client}.crt" + key_name = f"{cr.client_certs_dir}/{client}.key" + + conf_name = f"{cr.ca_key_dir}/conf.client.{client}" + csr_name = f"{cr.ca_key_dir}/csr.client.{client}" + + if os.path.exists(crt_name) and os.path.exists(key_name): # No need to regenerate it + continue + + with open(conf_name, "w") as f: + f.write( + f"""[ req ] +prompt=no +distinguished_name = my_distinguished_name + +[ my_distinguished_name ] +organizationName = Yugabyte +commonName = {client} +""" + ) + + subprocess.check_call(["openssl", "genrsa", "-out", key_name]) + + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-config", + conf_name, + "-key", + key_name, + "-out", + csr_name, + ] + ) + + subprocess.check_call( + [ + "openssl", + "ca", + "-config", + cr.ca_conf, + "-keyfile", + cr.ca_key_file, + "-cert", + cr.ca_cert_file, + "-policy", + "my_policy", + "-out", + crt_name, + "-outdir", + cr.client_certs_dir, + "-in", + csr_name, + "-days", + "3650", + "-batch", + "-extfile", + conf_name, + ] + ) + + ### SERVERS + + for server_type in ["master", "tserver"]: + + folder = getattr(cr, f"{server_type}_certs_dir") + + # Copy CA + shutil.copy(cr.ca_cert_file, folder) + + for server_id in range(0, int(args.node_count)): + short_name = f"yb-{server_type}-{server_id}" + short_name_group = f"{short_name}.yb-{server_type}s" + full_name_group = f"{short_name}.{cr.namespace}.svc.cluster.local" + full_name = f"{short_name_group}.{cr.namespace}.svc.cluster.local" + + crt_name = f"{folder}/node.{full_name}.crt" + key_name = f"{folder}/node.{full_name}.key" + + conf_name = f"{cr.ca_key_dir}/conf.{full_name}" + csr_name = f"{cr.ca_key_dir}/csr.{full_name}" + + if os.path.exists(crt_name) and os.path.exists(key_name): # No need to regenerate it + continue + + with open(conf_name, "w") as f: + f.write( + f"""[ req ] +prompt=no +distinguished_name = my_distinguished_name + +[ my_distinguished_name ] +organizationName = Yugabyte +commonName = {full_name} + +# Multiple subject alternative names (SANs) such as IP Address, +# DNS Name, Email, URI, and so on, can be specified under this section +[ req_ext] +subjectAltName = @alt_names +[alt_names] +DNS.1 = {short_name} +DNS.2 = {full_name} +DNS.3 = {short_name_group} +DNS.4 = {full_name_group} +DNS.5 = yb-{server_type}s +DNS.6 = yb-{server_type}s.{cr.namespace} +DNS.7 = yb-{server_type}s.{cr.namespace}.svc.cluster.local +""" + ) + + subprocess.check_call(["openssl", "genrsa", "-out", key_name]) + + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-config", + conf_name, + "-key", + key_name, + "-out", + csr_name, + ] + ) + + subprocess.check_call( + [ + "openssl", + "ca", + "-config", + cr.ca_conf, + "-keyfile", + cr.ca_key_file, + "-cert", + cr.ca_cert_file, + "-policy", + "my_policy", + "-out", + crt_name, + "-outdir", + folder, + "-in", + csr_name, + "-days", + "3650", + "-batch", + "-extfile", + conf_name, + "-extensions", + "req_ext", + ] + ) + + +if __name__ == "__main__": + main() diff --git a/build/make-certs.py b/build/make-certs.py index 2ea8f1b0e..1f54b8310 100755 --- a/build/make-certs.py +++ b/build/make-certs.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 import argparse -import itertools -import glob import os import shutil import subprocess diff --git a/deploy/services/helm-charts/dss/templates/_helpers.tpl b/deploy/services/helm-charts/dss/templates/_helpers.tpl index 67670520f..2939f9445 100644 --- a/deploy/services/helm-charts/dss/templates/_helpers.tpl +++ b/deploy/services/helm-charts/dss/templates/_helpers.tpl @@ -50,6 +50,6 @@ yugabyte {{ if .cockroachdb }} - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.databaseHost}} --port \"{{.databasePort}}\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}" {{ else }} - - "ysqlsh --host {{.databaseHost}} --port \"{{.databasePort}}\" -c \"SELECT datname FROM pg_database where datname = '{{.schemaName}}';\" | grep {{.schemaName}}" + - "ysqlsh --host {{.databaseHost}} --port \"{{.databasePort}}\" \"sslmode=require\" -c \"SELECT datname FROM pg_database where datname = '{{.schemaName}}';\" | grep {{.schemaName}}" {{ end }} {{- end -}} diff --git a/deploy/services/helm-charts/dss/templates/_volumes.tpl b/deploy/services/helm-charts/dss/templates/_volumes.tpl index e21dc30e4..e5af35405 100644 --- a/deploy/services/helm-charts/dss/templates/_volumes.tpl +++ b/deploy/services/helm-charts/dss/templates/_volumes.tpl @@ -2,27 +2,41 @@ - name: ca-certs secret: defaultMode: 256 - secretName: cockroachdb.ca.crt + secretName: {{ if .cockroachdb }}cockroachdb.ca.crt{{ else }}yugabyte-tls-client-cert{{ end }} {{- end -}} {{- define "ca-certs:volumeMount" -}} +{{ if .cockroachdb }} - mountPath: /cockroach/cockroach-certs/ca.crt name: ca-certs subPath: ca.crt +{{ else }} +- mountPath: /opt/yugabyte-certs/ca.crt + name: ca-certs + subPath: root.crt +{{- end -}} {{- end -}} - {{- define "client-certs:volume" -}} - name: client-certs secret: defaultMode: 256 - secretName: cockroachdb.client.root + secretName: {{ if .cockroachdb }}cockroachdb.client.root{{ else }}yugabyte-tls-client-cert{{ end }} {{- end -}} {{- define "client-certs:volumeMount" -}} +{{ if .cockroachdb }} - mountPath: /cockroach/cockroach-certs/client.root.crt name: client-certs subPath: client.root.crt - mountPath: /cockroach/cockroach-certs/client.root.key name: client-certs subPath: client.root.key +{{ else }} +- mountPath: /opt/yugabyte-certs/client.yugabyte.crt + name: client-certs + subPath: yugabytedb.crt +- mountPath: /opt/yugabyte-certs/client.yugabyte.key + name: client-certs + subPath: yugabytedb.key +{{- end -}} {{- end -}} @@ -35,4 +49,4 @@ {{- define "public-certs:volumeMount" -}} - mountPath: /public-certs name: public-certs -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 625ca470f..67f2d376e 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -51,6 +51,9 @@ spec: - --cockroach_ssl_dir=/cockroach/cockroach-certs - --cockroach_ssl_mode=verify-full - --locality={{ .Values.cockroachdb.conf.locality }} +{{ else }} + - --cockroach_ssl_dir=/opt/yugabyte-certs + - --cockroach_ssl_mode=verify-full {{ end }} - --dump_requests=true - --enable_scd={{$dss.enableScd | default true}} @@ -81,12 +84,12 @@ spec: stdin: false tty: false volumeMounts: - {{- include "ca-certs:volumeMount" . | nindent 12 }} - {{- include "client-certs:volumeMount" . | nindent 12 }} - {{- include "public-certs:volumeMount" . | nindent 12 }} + {{- include "ca-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "client-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "public-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} imagePullSecrets: [] terminationGracePeriodSeconds: 30 volumes: - {{- include "ca-certs:volume" . | nindent 8 }} - {{- include "client-certs:volume" . | nindent 8 }} - {{- include "public-certs:volume" . | nindent 8 }} + {{- include "ca-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "client-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "public-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} diff --git a/deploy/services/helm-charts/dss/templates/schema-manager.yaml b/deploy/services/helm-charts/dss/templates/schema-manager.yaml index 00459a989..8ff2a8dec 100644 --- a/deploy/services/helm-charts/dss/templates/schema-manager.yaml +++ b/deploy/services/helm-charts/dss/templates/schema-manager.yaml @@ -46,6 +46,8 @@ spec: - --schemas_dir=/db-schemas/{{$service}} {{ else }} - --schemas_dir=/db-schemas/yugabyte/{{$service}} + - --cockroach_ssl_dir=/opt/yugabyte-certs/ + - --cockroach_ssl_mode=verify-full {{ end }} - --db_version={{$schemaVersion}} command: @@ -57,12 +59,12 @@ spec: stdin: false tty: false volumeMounts: - {{- include "ca-certs:volumeMount" . | nindent 12 }} - {{- include "client-certs:volumeMount" . | nindent 12 }} + {{- include "ca-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "client-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} imagePullSecrets: [] restartPolicy: OnFailure terminationGracePeriodSeconds: 30 volumes: - {{- include "ca-certs:volume" . | nindent 8 }} - {{- include "client-certs:volume" . | nindent 8 }} + {{- include "ca-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "client-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} {{- end -}} diff --git a/deploy/services/helm-charts/dss/values.yaml b/deploy/services/helm-charts/dss/values.yaml index dfb2ed0e2..0d4727efd 100644 --- a/deploy/services/helm-charts/dss/values.yaml +++ b/deploy/services/helm-charts/dss/values.yaml @@ -19,4 +19,27 @@ yugabyte: Image: repository: yugabytedb/yugabyte + isMultiAz: true + # multicluster: + # createServicePerPod: true + + gflags: + master: + placement_cloud: "dss" + placement_region: "uss-1" + placement_zone: "zone" + tserver: + placement_cloud: "dss" + placement_region: "uss-1" + placement_zone: "zone" + + masterAddresses: "yb-master-0.yb-masters.default.svc.cluster.local:7100,yb-master-1.yb-masters.default.svc.cluster.local:7100,yb-master-2.yb-masters.default.svc.cluster.local:7100" + + tls: + enabled: true + nodeToNode: true + clientToServer: true + insecure: false + provided: true + # See https://github.com/yugabyte/charts/blob/master/stable/yugabyte/values.yaml From 87a54792475eb5bee64109e896650dd1347b5a25 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 6 May 2025 14:17:01 +0200 Subject: [PATCH 07/18] Fixes for PR --- .../helm-charts/dss/templates/_helpers.tpl | 16 ++++++------- .../dss/templates/dss-core-service.yaml | 24 +++++++++---------- .../dss/templates/schema-manager.yaml | 18 +++++++------- .../helm-charts/dss/values.example.yaml | 4 ++-- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/deploy/services/helm-charts/dss/templates/_helpers.tpl b/deploy/services/helm-charts/dss/templates/_helpers.tpl index 67670520f..d0f4f2243 100644 --- a/deploy/services/helm-charts/dss/templates/_helpers.tpl +++ b/deploy/services/helm-charts/dss/templates/_helpers.tpl @@ -1,4 +1,4 @@ -{{- define "databaseImage" -}} +{{- define "datastoreImage" -}} {{- if $.Values.cockroachdb.enabled -}} {{ (printf "%s:%s" $.Values.cockroachdb.image.repository $.Values.cockroachdb.image.tag) }} {{- else -}} @@ -6,7 +6,7 @@ {{- end -}} {{- end -}} -{{- define "databasePort" -}} +{{- define "datastorePort" -}} {{- if $.Values.cockroachdb.enabled -}} 26257 {{- else -}} @@ -14,7 +14,7 @@ {{- end -}} {{- end -}} -{{- define "databaseUser" -}} +{{- define "datastoreUser" -}} {{- if $.Values.cockroachdb.enabled -}} root {{- else -}} @@ -23,7 +23,7 @@ yugabyte {{- end -}} -{{- define "databaseHost" -}} +{{- define "datastoreHost" -}} {{- if $.Values.cockroachdb.enabled -}} {{- printf "%s-public.default" $.Values.cockroachdb.fullnameOverride -}} {{- else -}} @@ -38,9 +38,9 @@ yugabyte {{- end -}} {{- define "init-container-wait-for-schema" -}} -{{/*For some reason, calling the template databaseImage fails here.*/}} +{{/*For some reason, calling the template datastoreImage fails here.*/}} - name: wait-for-schema-{{.schemaName}} - image: {{.databaseImage}} + image: {{.datastoreImage}} volumeMounts: {{- include "ca-certs:volumeMount" . | nindent 4 }} {{- include "client-certs:volumeMount" . | nindent 4 }} @@ -48,8 +48,8 @@ yugabyte - sh - -c {{ if .cockroachdb }} - - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.databaseHost}} --port \"{{.databasePort}}\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}" + - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.datastoreHost}} --port \"{{.datastorePort}}\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}" {{ else }} - - "ysqlsh --host {{.databaseHost}} --port \"{{.databasePort}}\" -c \"SELECT datname FROM pg_database where datname = '{{.schemaName}}';\" | grep {{.schemaName}}" + - "ysqlsh --host {{.datastoreHost}} --port \"{{.datastorePort}}\" -c \"SELECT datname FROM pg_database where datname = '{{.schemaName}}';\" | grep {{.schemaName}}" {{ end }} {{- end -}} diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 625ca470f..273eaee64 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -1,17 +1,17 @@ {{- $dss := .Values.dss -}} -{{- $databaseImage := (include "databaseImage" .) -}} -{{- $databaseHost := (include "databaseHost" .) -}} -{{- $databasePort := (include "databasePort" .) -}} -{{- $databaseUser := (include "databaseUser" .) -}} +{{- $datastoreImage := (include "datastoreImage" .) -}} +{{- $datastoreHost := (include "datastoreHost" .) -}} +{{- $datastorePort := (include "datastorePort" .) -}} +{{- $datastoreUser := (include "datastoreUser" .) -}} -{{- $waitForDatabase := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $databaseHost)) -}} +{{- $waitForDatastore := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $datastoreHost)) -}} {{- if .Values.yugabyte.enabled }} -{{- $waitForDatabase = include "init-container-wait-for-http" (dict "serviceName" "yb-tserver" "url" (printf "http://%s:9000/status" $databaseHost)) -}} +{{- $waitForDatastore = include "init-container-wait-for-http" (dict "serviceName" "yb-tserver" "url" (printf "http://%s:9000/status" $datastoreHost)) -}} {{- end -}} -{{- $waitForRIDSchema := include "init-container-wait-for-schema" (dict "schemaName" "rid" "databaseImage" $databaseImage "databasePort" $databasePort "databaseHost" $databaseHost "cockroachdb" .Values.cockroachdb.enabled ) -}} -{{- $waitForSCDSchema := include "init-container-wait-for-schema" (dict "schemaName" "scd" "databaseImage" $databaseImage "databasePort" $databasePort "databaseHost" $databaseHost "cockroachdb" .Values.cockroachdb.enabled ) -}} +{{- $waitForRIDSchema := include "init-container-wait-for-schema" (dict "schemaName" "rid" "datastoreImage" $datastoreImage "datastorePort" $datastorePort "datastoreHost" $datastoreHost "cockroachdb" .Values.cockroachdb.enabled ) -}} +{{- $waitForSCDSchema := include "init-container-wait-for-schema" (dict "schemaName" "scd" "datastoreImage" $datastoreImage "datastorePort" $datastorePort "datastoreHost" $datastoreHost "cockroachdb" .Values.cockroachdb.enabled ) -}} --- apiVersion: apps/v1 @@ -37,16 +37,16 @@ spec: app: {{.Release.Name}}-core-service spec: initContainers: - {{- $waitForDatabase | nindent 8 }} + {{- $waitForDatastore | nindent 8 }} {{- $waitForRIDSchema | nindent 8 }} {{- $waitForSCDSchema | nindent 8 }} containers: - args: - --accepted_jwt_audiences={{$dss.conf.hostname}} - --addr=:8080 - - --cockroach_host={{ $databaseHost }} - - --cockroach_port={{ $databasePort }} - - --cockroach_user={{ $databaseUser }} + - --cockroach_host={{ $datastoreHost }} + - --cockroach_port={{ $datastorePort }} + - --cockroach_user={{ $datastoreUser }} {{ if $.Values.cockroachdb.enabled }} - --cockroach_ssl_dir=/cockroach/cockroach-certs - --cockroach_ssl_mode=verify-full diff --git a/deploy/services/helm-charts/dss/templates/schema-manager.yaml b/deploy/services/helm-charts/dss/templates/schema-manager.yaml index 00459a989..fbd9b22a6 100644 --- a/deploy/services/helm-charts/dss/templates/schema-manager.yaml +++ b/deploy/services/helm-charts/dss/templates/schema-manager.yaml @@ -1,14 +1,14 @@ {{- $image := .Values.dss.image }} -{{- $databaseHost := (include "databaseHost" .) -}} -{{- $databasePort := (include "databasePort" .) -}} -{{- $databaseUser := (include "databaseUser" .) -}} +{{- $datastoreHost := (include "datastoreHost" .) -}} +{{- $datastorePort := (include "datastorePort" .) -}} +{{- $datastoreUser := (include "datastoreUser" .) -}} {{- $jobVersion := .Release.Revision -}} {{/* Jobs template definition is immutable, using the revision in the name forces the job to be recreated at each helm upgrade. */}} -{{- $waitForDatabase := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $databaseHost)) -}} +{{- $waitForDatastore := include "init-container-wait-for-http" (dict "serviceName" "cockroachdb" "url" (printf "http://%s:8080/health" $datastoreHost)) -}} {{- $schemas := dict "rid" "4.0.0" "scd" "3.2.0" }} {{- if .Values.yugabyte.enabled }} -{{- $waitForDatabase = include "init-container-wait-for-http" (dict "serviceName" "yb-tserver" "url" (printf "http://%s:9000/status" $databaseHost)) -}} +{{- $waitForDatastore = include "init-container-wait-for-http" (dict "serviceName" "yb-tserver" "url" (printf "http://%s:9000/status" $datastoreHost)) -}} {{- $schemas = dict "rid" "1.0.1" "scd" "1.0.1" }} {{- end -}} @@ -33,13 +33,13 @@ spec: name: {{$service}}-schema-manager-{{$jobVersion}} spec: initContainers: - {{- $waitForDatabase | nindent 8 }} + {{- $waitForDatastore | nindent 8 }} containers: - args: - migrate - - --cockroach_host={{ $databaseHost }} - - --cockroach_port={{ $databasePort }} - - --cockroach_user={{ $databaseUser }} + - --cockroach_host={{ $datastoreHost }} + - --cockroach_port={{ $datastorePort }} + - --cockroach_user={{ $datastoreUser }} {{ if $.Values.cockroachdb.enabled }} - --cockroach_ssl_dir=/cockroach/cockroach-certs - --cockroach_ssl_mode=verify-full diff --git a/deploy/services/helm-charts/dss/values.example.yaml b/deploy/services/helm-charts/dss/values.example.yaml index da1321c8e..1395cb7bb 100644 --- a/deploy/services/helm-charts/dss/values.example.yaml +++ b/deploy/services/helm-charts/dss/values.example.yaml @@ -13,7 +13,7 @@ dss: enableScd: true cockroachdb: - enabled: false + enabled: true # See https://github.com/cockroachdb/helm-charts/blob/master/cockroachdb/values.yaml image: tag: v24.1.3 @@ -38,7 +38,7 @@ cockroachdb: storageClass: standard yugabyte: - enabled: true + enabled: false # See https://github.com/yugabyte/charts/blob/master/stable/yugabyte/values.yaml Image: tag: 2.25.1.0-b381 From 4fe7286560e810a4cd9093c605314e35d8d02819 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 6 May 2025 14:35:02 +0200 Subject: [PATCH 08/18] Fixes for PR --- deploy/services/helm-charts/dss/templates/_helpers.tpl | 2 +- .../services/helm-charts/dss/templates/dss-core-service.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/services/helm-charts/dss/templates/_helpers.tpl b/deploy/services/helm-charts/dss/templates/_helpers.tpl index d0f4f2243..f2e569003 100644 --- a/deploy/services/helm-charts/dss/templates/_helpers.tpl +++ b/deploy/services/helm-charts/dss/templates/_helpers.tpl @@ -47,7 +47,7 @@ yugabyte command: - sh - -c -{{ if .cockroachdb }} +{{ if .cockroachdbEnabled }} - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.datastoreHost}} --port \"{{.datastorePort}}\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}" {{ else }} - "ysqlsh --host {{.datastoreHost}} --port \"{{.datastorePort}}\" -c \"SELECT datname FROM pg_database where datname = '{{.schemaName}}';\" | grep {{.schemaName}}" diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 273eaee64..aa201801f 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -10,8 +10,8 @@ {{- $waitForDatastore = include "init-container-wait-for-http" (dict "serviceName" "yb-tserver" "url" (printf "http://%s:9000/status" $datastoreHost)) -}} {{- end -}} -{{- $waitForRIDSchema := include "init-container-wait-for-schema" (dict "schemaName" "rid" "datastoreImage" $datastoreImage "datastorePort" $datastorePort "datastoreHost" $datastoreHost "cockroachdb" .Values.cockroachdb.enabled ) -}} -{{- $waitForSCDSchema := include "init-container-wait-for-schema" (dict "schemaName" "scd" "datastoreImage" $datastoreImage "datastorePort" $datastorePort "datastoreHost" $datastoreHost "cockroachdb" .Values.cockroachdb.enabled ) -}} +{{- $waitForRIDSchema := include "init-container-wait-for-schema" (dict "schemaName" "rid" "datastoreImage" $datastoreImage "datastorePort" $datastorePort "datastoreHost" $datastoreHost "cockroachdbEnabled" .Values.cockroachdb.enabled ) -}} +{{- $waitForSCDSchema := include "init-container-wait-for-schema" (dict "schemaName" "scd" "datastoreImage" $datastoreImage "datastorePort" $datastorePort "datastoreHost" $datastoreHost "cockroachdbEnabled" .Values.cockroachdb.enabled ) -}} --- apiVersion: apps/v1 From 7a0176fc5be9e210d1856c26526a3d20636e6eb5 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 6 May 2025 14:40:42 +0200 Subject: [PATCH 09/18] Normalize cockroachdbEnabled from previous PR --- .../services/helm-charts/dss/templates/_volumes.tpl | 8 ++++---- .../helm-charts/dss/templates/dss-core-service.yaml | 12 ++++++------ .../helm-charts/dss/templates/schema-manager.yaml | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/deploy/services/helm-charts/dss/templates/_volumes.tpl b/deploy/services/helm-charts/dss/templates/_volumes.tpl index e5af35405..981e5d80c 100644 --- a/deploy/services/helm-charts/dss/templates/_volumes.tpl +++ b/deploy/services/helm-charts/dss/templates/_volumes.tpl @@ -2,10 +2,10 @@ - name: ca-certs secret: defaultMode: 256 - secretName: {{ if .cockroachdb }}cockroachdb.ca.crt{{ else }}yugabyte-tls-client-cert{{ end }} + secretName: {{ if .cockroachdbEnabled }}cockroachdb.ca.crt{{ else }}yugabyte-tls-client-cert{{ end }} {{- end -}} {{- define "ca-certs:volumeMount" -}} -{{ if .cockroachdb }} +{{ if .cockroachdbEnabled }} - mountPath: /cockroach/cockroach-certs/ca.crt name: ca-certs subPath: ca.crt @@ -19,10 +19,10 @@ - name: client-certs secret: defaultMode: 256 - secretName: {{ if .cockroachdb }}cockroachdb.client.root{{ else }}yugabyte-tls-client-cert{{ end }} + secretName: {{ if .cockroachdbEnabled }}cockroachdb.client.root{{ else }}yugabyte-tls-client-cert{{ end }} {{- end -}} {{- define "client-certs:volumeMount" -}} -{{ if .cockroachdb }} +{{ if .cockroachdbEnabled }} - mountPath: /cockroach/cockroach-certs/client.root.crt name: client-certs subPath: client.root.crt diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 88360178c..e9ad8578a 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -84,12 +84,12 @@ spec: stdin: false tty: false volumeMounts: - {{- include "ca-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} - {{- include "client-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} - {{- include "public-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "ca-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "client-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "public-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} imagePullSecrets: [] terminationGracePeriodSeconds: 30 volumes: - {{- include "ca-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} - {{- include "client-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} - {{- include "public-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "ca-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "client-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "public-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} diff --git a/deploy/services/helm-charts/dss/templates/schema-manager.yaml b/deploy/services/helm-charts/dss/templates/schema-manager.yaml index 6376723e0..f5a232d76 100644 --- a/deploy/services/helm-charts/dss/templates/schema-manager.yaml +++ b/deploy/services/helm-charts/dss/templates/schema-manager.yaml @@ -59,12 +59,12 @@ spec: stdin: false tty: false volumeMounts: - {{- include "ca-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} - {{- include "client-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "ca-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "client-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} imagePullSecrets: [] restartPolicy: OnFailure terminationGracePeriodSeconds: 30 volumes: - {{- include "ca-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} - {{- include "client-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "ca-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "client-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} {{- end -}} From f24ccab5ef91802f3e1602ee1768cbe09734b7b0 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 6 May 2025 17:55:52 +0200 Subject: [PATCH 10/18] Remove duplicate locality --- deploy/services/helm-charts/dss/templates/dss-core-service.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index aa201801f..d129031ed 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -62,7 +62,6 @@ spec: {{- if $dss.conf.jwksKeyIds }} - --jwks_key_ids={{ $dss.conf.jwksKeyIds | join "," }} {{- end }} - - --locality={{ .Values.cockroachdb.conf.locality }} {{- if $dss.conf.pubKeys }} - --public_key_files={{ $dss.conf.pubKeys | join "," }} {{- end }} From 63f86f240d1b17362dded0cab7bfe83b231a32b5 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Wed, 7 May 2025 20:23:08 +0200 Subject: [PATCH 11/18] Yugabyte: Certificate managment --- .gitignore | 2 + build/apply-certs-yugabyte.sh | 54 --- build/make-certs-yugabyte.py | 364 ------------------ .../certificates-management/README.md | 131 +++++++ .../certificates-management/__init__.py | 0 .../certificates-management/apply.py | 52 +++ .../certificates-management/ca_pool.py | 144 +++++++ .../certificates-management/cluster.py | 135 +++++++ .../certificates-management/dss-certs.py | 143 +++++++ .../certificates-management/init.py | 240 ++++++++++++ .../certificates-management/nodes.py | 152 ++++++++ .../certificates-management/utils.py | 51 +++ 12 files changed, 1050 insertions(+), 418 deletions(-) delete mode 100755 build/apply-certs-yugabyte.sh delete mode 100755 build/make-certs-yugabyte.py create mode 100644 deploy/operations/certificates-management/README.md create mode 100644 deploy/operations/certificates-management/__init__.py create mode 100644 deploy/operations/certificates-management/apply.py create mode 100644 deploy/operations/certificates-management/ca_pool.py create mode 100644 deploy/operations/certificates-management/cluster.py create mode 100755 deploy/operations/certificates-management/dss-certs.py create mode 100644 deploy/operations/certificates-management/init.py create mode 100644 deploy/operations/certificates-management/nodes.py create mode 100644 deploy/operations/certificates-management/utils.py diff --git a/.gitignore b/.gitignore index 689976da1..4dda7a12b 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,8 @@ build/cockroachdb.yaml build/values.yaml build/dss.yaml +deploy/operations/certificates-management/workspace/ + temp # Django stuff: diff --git a/build/apply-certs-yugabyte.sh b/build/apply-certs-yugabyte.sh deleted file mode 100755 index 8bcb72cb4..000000000 --- a/build/apply-certs-yugabyte.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -if [ "$1" == "" ]; then - echo "usage: ./apply-certs.sh " - exit 1 -fi - -if [ "$2" == "" ]; then - echo "usage: ./apply-certs.sh " - exit 1 -fi - -echo '=========================================================================' -echo '= Note that errors below are acceptable as long as the terminal message =' -echo '= is success. =' -echo '=========================================================================' - -set -e -set -x - -# Paths to directories in which to store certificates and generated YAML files. -CONTEXT="$1" -DIR="$(pwd)" -NAMESPACE="$2" - -# Replace characters breaking folder names -WORKSPACE=$(echo "${CONTEXT}" | tr ':/' '_') -CLIENTS_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/client_certs_dir" -MASTER_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/master_certs_dir" -TSERVER_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/tserver_certs_dir" -# CA_KEY_DIR="$DIR/workspace/$WORKSPACE/ca_key_dir" -# CA_CRT_DIR="$DIR/workspace/$WORKSPACE/ca_certs_dir" -JWT_PUBLIC_CERTS_DIR="$DIR/jwt-public-certs" - -# Delete previous secrets in case they have changed. -kubectl create namespace "$NAMESPACE" --context "$CONTEXT" || true - -kubectl delete secret yb-master-yugabyte-tls-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true -kubectl delete secret yb-tserver-yugabyte-tls-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true -kubectl delete secret yugabyte-tls-client-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true -kubectl delete secret dss.public.certs --namespace "$NAMESPACE" --context "$CONTEXT" || true - -kubectl create secret generic yb-master-yugabyte-tls-cert --namespace "$NAMESPACE" --from-file "$MASTER_CERTS_DIR" --context "$CONTEXT" -kubectl create secret generic yb-tserver-yugabyte-tls-cert --namespace "$NAMESPACE" --from-file "$TSERVER_CERTS_DIR" --context "$CONTEXT" -kubectl create secret generic yugabyte-tls-client-cert --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" - - -kubectl create secret generic dss.public.certs --namespace "$NAMESPACE" --from-file "$JWT_PUBLIC_CERTS_DIR" --context "$CONTEXT" - -echo '=========================================================================' -echo '= Secrets uploaded successfully. =' -echo '=========================================================================' diff --git a/build/make-certs-yugabyte.py b/build/make-certs-yugabyte.py deleted file mode 100755 index 3f209999e..000000000 --- a/build/make-certs-yugabyte.py +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import shutil -import subprocess - - -class YugabyteCluster(object): - - def __init__(self, cluster_context, namespace, ca_cert_to_join=None): - self._ca_cert_to_join = ca_cert_to_join - self._cluster_context = cluster_context - self._namespace = namespace - - @property - def ca_cert_to_join(self): - return self._ca_cert_to_join - - @property - def namespace(self): - return self._namespace - - @property - def directory(self): - # Replace characters breaking folder names - def remove_special_chars(s: str): - for c in [":", "/"]: - s = s.replace(c, "_") - return s - - return os.path.join( - os.getcwd(), - "workspace-yugabyte", remove_special_chars(self._cluster_context) - ) - - @property - def ca_key_dir(self): - return os.path.join(self.directory, "ca_key_dir") - - @property - def ca_key_file(self): - return os.path.join(self.ca_key_dir, "ca.key") - - @property - def ca_cert_file(self): - return os.path.join(self.ca_key_dir, "ca.crt") - - @property - def ca_conf(self): - return os.path.join(self.ca_key_dir, "ca.conf") - - @property - def ca_certs_dir(self): - return os.path.join(self.directory, "ca_certs_dir") - - @property - def client_certs_dir(self): - return os.path.join(self.directory, "client_certs_dir") - - @property - def master_certs_dir(self): - return os.path.join(self.directory, "master_certs_dir") - - @property - def tserver_certs_dir(self): - return os.path.join(self.directory, "tserver_certs_dir") - - -def parse_args(): - parser = argparse.ArgumentParser( - description="Creates certificates for a new Cockroachdb cluster" - ) - parser.add_argument( - "--cluster-context", - metavar="CLUSTER_CONTEXT", - required=True, - help="kubernetes cluster context name", - ) - parser.add_argument( - "--namespace", - metavar="NAMESPACE", - required=True, - help="kubernetes cluster namespace you are deploying to.", - ) - # TODO - # parser.add_argument( - # "--node-address", - # metavar="ADDRESS", - # nargs="*", - # default=[], - # help="extra addresses to add to the node certificate", - # ) - parser.add_argument( - "--ca-cert-to-join", - metavar="FILENAME", - help="file containing an existing CA cert of a cluster to join.", - ) - parser.add_argument( - "--overwrite-ca-cert", - action="store_true", - default=False, - help="True to generate new CA certs, false to use the existing one", - ) - parser.add_argument( - "--node-count", - metavar="NODE_COUNT", - default="3", - help="Number of yugabyte nodes in the cluster, default to 3", - ) - return parser.parse_args() - - -def main(): - args = parse_args() - cr = YugabyteCluster(args.cluster_context, args.namespace, args.ca_cert_to_join) - - # Create the generated directories. - if not os.path.exists("workspace-yugabyte"): - os.makedirs("workspace-yugabyte") - - if not os.path.exists(cr.directory): - os.makedirs(cr.directory) - - create_ca = not os.path.exists(cr.ca_key_dir) or args.overwrite_ca_cert - if create_ca: - # Create a new CA. - # Delete and recreate the ca_certs_dir. - shutil.rmtree(cr.ca_key_dir, ignore_errors=True) - os.mkdir(cr.ca_key_dir) - - # Build master, tserver and client certs. - os.mkdir(cr.master_certs_dir) - os.mkdir(cr.tserver_certs_dir) - os.mkdir(cr.client_certs_dir) - - if create_ca: - - with open(cr.ca_conf, "w") as f: - f.write( - f""" - [ ca ] - default_ca = my_ca - -[ my_ca ] -default_days = 3650 - -serial = {cr.ca_key_dir}/serial.txt -database = {cr.ca_key_dir}/index.txt -default_md = sha256 -policy = my_policy - -[ my_policy ] - -organizationName = supplied -commonName = supplied - -[req] -prompt=no -distinguished_name = my_distinguished_name -x509_extensions = my_extensions - -[ my_distinguished_name ] -organizationName = Yugabyte -commonName = CA for YugabyteDB - -[ my_extensions ] -keyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment,keyCertSign -basicConstraints = critical,CA:true,pathlen:1 - -""" - ) - - with open(f"{cr.ca_key_dir}/serial.txt", "w") as f: - f.write("01") - - with open(f"{cr.ca_key_dir}/index.txt", "w") as f: - f.write("") - - subprocess.check_call(["openssl", "genrsa", "-out", cr.ca_key_file]) - - subprocess.check_call( - [ - "openssl", - "req", - "-new", - "-x509", - "-days", - "3650", - "-config", - cr.ca_conf, - "-key", - cr.ca_key_file, - "-out", - cr.ca_cert_file, - ] - ) - - ### CLIENT CERTIFICATE - - # Copy CA - shutil.copy(cr.ca_cert_file, f"{cr.client_certs_dir}/root.crt") - - for client in ["yugabytedb"]: - - crt_name = f"{cr.client_certs_dir}/{client}.crt" - key_name = f"{cr.client_certs_dir}/{client}.key" - - conf_name = f"{cr.ca_key_dir}/conf.client.{client}" - csr_name = f"{cr.ca_key_dir}/csr.client.{client}" - - if os.path.exists(crt_name) and os.path.exists(key_name): # No need to regenerate it - continue - - with open(conf_name, "w") as f: - f.write( - f"""[ req ] -prompt=no -distinguished_name = my_distinguished_name - -[ my_distinguished_name ] -organizationName = Yugabyte -commonName = {client} -""" - ) - - subprocess.check_call(["openssl", "genrsa", "-out", key_name]) - - subprocess.check_call( - [ - "openssl", - "req", - "-new", - "-config", - conf_name, - "-key", - key_name, - "-out", - csr_name, - ] - ) - - subprocess.check_call( - [ - "openssl", - "ca", - "-config", - cr.ca_conf, - "-keyfile", - cr.ca_key_file, - "-cert", - cr.ca_cert_file, - "-policy", - "my_policy", - "-out", - crt_name, - "-outdir", - cr.client_certs_dir, - "-in", - csr_name, - "-days", - "3650", - "-batch", - "-extfile", - conf_name, - ] - ) - - ### SERVERS - - for server_type in ["master", "tserver"]: - - folder = getattr(cr, f"{server_type}_certs_dir") - - # Copy CA - shutil.copy(cr.ca_cert_file, folder) - - for server_id in range(0, int(args.node_count)): - short_name = f"yb-{server_type}-{server_id}" - short_name_group = f"{short_name}.yb-{server_type}s" - full_name_group = f"{short_name}.{cr.namespace}.svc.cluster.local" - full_name = f"{short_name_group}.{cr.namespace}.svc.cluster.local" - - crt_name = f"{folder}/node.{full_name}.crt" - key_name = f"{folder}/node.{full_name}.key" - - conf_name = f"{cr.ca_key_dir}/conf.{full_name}" - csr_name = f"{cr.ca_key_dir}/csr.{full_name}" - - if os.path.exists(crt_name) and os.path.exists(key_name): # No need to regenerate it - continue - - with open(conf_name, "w") as f: - f.write( - f"""[ req ] -prompt=no -distinguished_name = my_distinguished_name - -[ my_distinguished_name ] -organizationName = Yugabyte -commonName = {full_name} - -# Multiple subject alternative names (SANs) such as IP Address, -# DNS Name, Email, URI, and so on, can be specified under this section -[ req_ext] -subjectAltName = @alt_names -[alt_names] -DNS.1 = {short_name} -DNS.2 = {full_name} -DNS.3 = {short_name_group} -DNS.4 = {full_name_group} -DNS.5 = yb-{server_type}s -DNS.6 = yb-{server_type}s.{cr.namespace} -DNS.7 = yb-{server_type}s.{cr.namespace}.svc.cluster.local -""" - ) - - subprocess.check_call(["openssl", "genrsa", "-out", key_name]) - - subprocess.check_call( - [ - "openssl", - "req", - "-new", - "-config", - conf_name, - "-key", - key_name, - "-out", - csr_name, - ] - ) - - subprocess.check_call( - [ - "openssl", - "ca", - "-config", - cr.ca_conf, - "-keyfile", - cr.ca_key_file, - "-cert", - cr.ca_cert_file, - "-policy", - "my_policy", - "-out", - crt_name, - "-outdir", - folder, - "-in", - csr_name, - "-days", - "3650", - "-batch", - "-extfile", - conf_name, - "-extensions", - "req_ext", - ] - ) - - -if __name__ == "__main__": - main() diff --git a/deploy/operations/certificates-management/README.md b/deploy/operations/certificates-management/README.md new file mode 100644 index 000000000..09a0bdf7c --- /dev/null +++ b/deploy/operations/certificates-management/README.md @@ -0,0 +1,131 @@ +# Certificates management + +## Introduction + +The `dss-certs.py` helps you manage the set of certificates used for your DSS deployment. + +Should this DSS beeing part of a pool, the script also provide some helpers to manage the set of CA certificates in the pool. + +To run the script, just run `./dss-certs.py`. The python script don't require any dependencies, just a recent version of python 3. + +## Quick start guide + +### Simple local cluster in minikube` + +* `./dss-certs.py --name test --cluster-context dss-local-cluster --namespace default init` +* `./dss-certs.py --name test --cluster-context dss-local-cluster --namespace default apply` + +### Simple pool of 3 local cluster in minikube, in namespace `default`, `ns2` and `ns3` + +* Creation of the 3 cluster's certificates +* `./dss-certs.py --name localpool --cluster-context dss-local-cluster --namespace default init` +* `./dss-certs.py --name localpool2 --cluster-context dss-local-cluster --namespace ns2 init` +* `./dss-certs.py --name localpool3 --cluster-context dss-local-cluster --namespace ns3 init` +* Copy of cluster 2 and 3 CA to the base cluster +* `./dss-certs.py --name localpool2 --cluster-context dss-local-cluster --namespace ns2 get-ca | ./dss-certs.py --name localpool --cluster-context dss-local-cluster --namespace default add-pool-ca` +* `./dss-certs.py --name localpool3 --cluster-context dss-local-cluster --namespace ns3 get-ca | ./dss-certs.py --name localpool --cluster-context dss-local-cluster --namespace default add-pool-ca` +* Copy of base cluster's CA pool to cluster 2 and 3's CA pool +* `./dss-certs.py --name localpool --cluster-context dss-local-cluster --namespace default get-pool-ca | ./dss-certs.py --name localpool2 --cluster-context dss-local-cluster --namespace ns2 add-pool-ca` +* `./dss-certs.py --name localpool --cluster-context dss-local-cluster --namespace default get-pool-ca | ./dss-certs.py --name localpool3 --cluster-context dss-local-cluster --namespace ns3 add-pool-ca` +* Application of certificates in respective clusters +* `./dss-certs.py --name localpool --cluster-context dss-local-cluster --namespace default apply` +* `./dss-certs.py --name localpool2 --cluster-context dss-local-cluster --namespace ns2 apply` +* `./dss-certs.py --name localpool3 --cluster-context dss-local-cluster --namespace ns3 apply` + +## Operations + +### Common parameters + +#### `--name` + +The name of your cluster, that should identify it in a unique way. Used as main identifier for the set of certificates and in certificates but may be different inside a DSS pool. + +Example: `dss-west-1` + +#### `--organization` + +The name or your organization. Used in certificates generation. The combination of (name, organization) shall be unique in a cluster. + +Example: `interuss` + +#### `--cluster-context` + +The kubernetes context the script should use. + +Example: `dss-local-cluster` + +#### `--namespace` + +The kubernetes namespace to use. + +Example: `default` + +#### `--nodes-count` + +The number of yugabyte nodes you have. Default to `3`. + +### `init` + +Create a new set of certificates, with a CA, a client certificate and a certificate for each yugabyte node. + +### `apply` + +Apply the current set of certificate to the kubernetes cluster. Shall be ran after each modification of the certificates, like addition / removal of CA in the pool, new `nodes-count` parameter. + +### `regenerate-nodes` + +Generate missing nodes certificates. Useful if you want to add new nodes in your cluster. Don't forget to set the `nodes-count` parameters. + +### `add-pool-ca` + +Add the CA certificate(s) of another(s) USS in the pool of trusted certificates. +Existing certificates are not added again, so you may simply use the output of `get-pool-ca` from another USS. + +You can set the file with certificate(s) with `--ca-file` or use stdin. + +Don't forget to use the `apply` command to update certificate on your kubernetes cluster. + +Examples: + +* `./dss-certs.py --name test --cluster-context dss-local-cluster --namespace default add-pool-ca < /tmp/new-dss-ca` +* `./dss-certs.py --name test --cluster-context dss-local-cluster --namespace default --ca-file /tmp/new-dss-ca add-pool-ca` +* `./dss-certs.py --name test --cluster-context dss-local-cluster --namespace default get-pool-ca | ./dss-certs.py --name test2 --cluster-context dss-local-cluster --namespace namespace2 add-pool-ca` + +### `remove-pool-ca` + +Remove the CA certificate(s) of another(s) USS in the pool of trusted certificates. +Unknown certificates are not remove again. + +You can set the file with certificate(s) with `--ca-file`, use stdin or use `--ca-serial` to specify the serial / name of the certificate you want to remove. + +Don't forget to use the `apply` command to update certificate on your kubernetes cluster. + +Example: + +* `./dss-certs.py --name test --cluster-context dss-local-cluster --namespace default remove-pool-ca < /tmp/old-dss-ca` +* `./dss-certs.py --name test --cluster-context dss-local-cluster --namespace default --ca-file /tmp/old-dss-ca remove-pool-ca` +* `./dss-certs.py --name test --cluster-context dss-local-cluster --namespace default remove-pool-ca --ca-serial="SN=830ECFB0, O=generic-dss-organization, CN=CA.test"` +* `./dss-certs.py --name test --cluster-context dss-local-cluster --namespace default remove-pool-ca --ca-serial="830ECFB0` +* `./dss-certs.py --name test --cluster-context dss-local-cluster --namespace default remove-pool-ca --ca-serial="46548B7CC9699A7CFA54FF8FA85A619E830ECFB0` + +### `list-pool-ca` + +List the current CA certificates in the CA pool. + +Also display a 'hash' of CA serial, that you may use to compare others USS CA pool certificates list easily. + +### `get-pool-ca` + +Return all CA certificate in the current pool. + +Can be used for debugging or to synchronize the set of CA certificates in a pool with others USS. + +### `get-ca` + +Return your own CA certificate . + +Can be used for debugging or to synchronize the set of CA certificates in a pool with others USS. + +### `destroy` + +Destroy a certificate set. Be careful, there are no way to undo the command. diff --git a/deploy/operations/certificates-management/__init__.py b/deploy/operations/certificates-management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/deploy/operations/certificates-management/apply.py b/deploy/operations/certificates-management/apply.py new file mode 100644 index 000000000..37234d1b8 --- /dev/null +++ b/deploy/operations/certificates-management/apply.py @@ -0,0 +1,52 @@ +import subprocess +import os + +import logging +l = logging.getLogger(__name__) + +def do_apply(cluster): + + l.debug("Applying kubernetes configuration") + + l.debug(f"Creating namespace {cluster.namespace}") + + try: + subprocess.check_call( + ["kubectl", "create", "namespace", cluster.namespace, "--context", cluster.cluster_context], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + l.info(f"Created namespace {cluster.namespace}") + + except subprocess.CalledProcessError: # We do assume everything else works + l.debug(f"Namespace {cluster.namespace} already exists") + + for secret in ["yb-master-yugabyte-tls-cert", "yb-tserver-yugabyte-tls-cert", "yugabyte-tls-client-cert", "dss.public.certs"]: + + try: + subprocess.check_call( + ["kubectl", "delete", "secret", secret, "--namespace", cluster.namespace, "--context", cluster.cluster_context], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + l.info(f"Deleted old secret '{secret}'") + + except subprocess.CalledProcessError: # We do assume everything else works + l.debug(f"Secret '{secret}' not present on the cluster") + + for secret, folder in [ + ("yb-master-yugabyte-tls-cert", cluster.master_certs_dir), + ("yb-tserver-yugabyte-tls-cert", cluster.tserver_certs_dir), + ("yugabyte-tls-client-cert", cluster.client_certs_dir), + ("dss.public.certs", os.path.join("..", "..", "..", "build", "jwt-public-certs")), + ]: + + subprocess.check_call( + ["kubectl", "create", "secret", "generic", secret, "--namespace", cluster.namespace, "--context", cluster.cluster_context, "--from-file", folder], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + l.info(f"Created secret '{secret}'") diff --git a/deploy/operations/certificates-management/ca_pool.py b/deploy/operations/certificates-management/ca_pool.py new file mode 100644 index 000000000..6a770d8be --- /dev/null +++ b/deploy/operations/certificates-management/ca_pool.py @@ -0,0 +1,144 @@ +import base64 +import hashlib +import logging +import os +import re +import shutil +import tempfile + +from utils import get_cert_display_name, get_cert_serial + +l = logging.getLogger(__name__) + + +def build_pool_hash(cluster): + + CAs = [] + for f in os.listdir(cluster.ca_pool_dir): + + if f.endswith(".crt") and f != "ca.crt": + CAs.append(f.lower()) + + CAs = sorted(CAs) + + h = hashlib.sha256() + h.update((",".join(CAs)).encode("utf-8")) + + hashed = base64.b64encode(h.digest(), b"Aa").decode("utf-8") + + return f"{hashed[:5]}-{hashed[-10:-5]}" + + +def add_cas(cluster, certificate): + + folder = cluster.ca_pool_dir + + l.debug("Getting new CA metadata") + + with tempfile.NamedTemporaryFile(delete_on_close=False) as tf: + tf.write(certificate.encode("utf-8")) + tf.close() + + serial = get_cert_serial(tf.name) + name = get_cert_display_name(tf.name) + + filename = f"{serial}.crt" + + target_file = os.path.join(folder, filename) + + if os.path.exists(target_file): + l.info(f"CA {name} already present in the pool") + return + + l.info(f"Adding CA {name} in the pool") + + with open(target_file, "w") as f: + f.write(certificate) + + +def regenerate_ca_files(cluster): + + l.debug("Regenerating CA files from all CA in the pool") + + CAs = [] + for filename in os.listdir(cluster.ca_pool_dir): + + if filename.endswith(".crt") and filename != "ca.crt": + with open(os.path.join(cluster.ca_pool_dir, filename), "r") as f: + CAs.append(f.read()) + + CAs = sorted(CAs) + + with open(cluster.ca_pool_ca, "w") as f: + f.write("\n\n".join(CAs)) + + shutil.copy(cluster.ca_pool_ca, cluster.client_ca) + + for node_type in ["master", "tserver"]: + shutil.copy(cluster.ca_pool_ca, getattr(cluster, f"{node_type}_ca")) + + h = build_pool_hash(cluster) + + l.info(f"Regenerated CA files from the CA pool. Current pool hash: {h}") + + +def do_add_cas(cluster, certificates): + pattern = re.compile( + r"-----BEGIN CERTIFICATE-----\s*.+?\s*-----END CERTIFICATE-----", re.DOTALL + ) + for cert in pattern.findall(certificates): + add_cas(cluster, cert) + + regenerate_ca_files(cluster) + + +def do_remove_cas(cluster, certificates_or_serial): + pattern = re.compile( + r"-----BEGIN CERTIFICATE-----\s*.+?\s*-----END CERTIFICATE-----", re.DOTALL + ) + for cert in pattern.findall(certificates_or_serial): + with tempfile.NamedTemporaryFile(delete_on_close=False) as tf: + tf.write(cert.encode("utf-8")) + tf.close() + serial = get_cert_serial(tf.name) + name = get_cert_display_name(tf.name) + + filename = f"{serial}.crt" + + target = os.path.join(cluster.ca_pool_dir, filename) + + if os.path.isfile(target): + os.unlink(target) + l.info(f"Removed certificate {name}") + else: + l.info(f"Certificate {name} not present in pool") + + for filename in sorted(os.listdir(cluster.ca_pool_dir)): + if filename.endswith(".crt") and filename != "ca.crt": + + serial = get_cert_serial(os.path.join(cluster.ca_pool_dir, filename)) + name = get_cert_display_name(os.path.join(cluster.ca_pool_dir, filename)) + + if certificates_or_serial == name or certificates_or_serial == serial or f"SN={certificates_or_serial}, " in name or name.startswith(certificates_or_serial): + os.unlink(os.path.join(cluster.ca_pool_dir, filename)) + l.info(f"Removed certificate {name}") + + regenerate_ca_files(cluster) + +def do_get_ca(cluster): + with open(cluster.ca_cert_file, "r") as f: + print(f.read()) + +def do_get_pool_ca(cluster): + with open(cluster.ca_pool_ca, "r") as f: + print(f.read()) + +def do_list_pool_ca(cluster): + + h = build_pool_hash(cluster) + + print(f"Current CA pool hash: {h}") + + for filename in sorted(os.listdir(cluster.ca_pool_dir)): + if filename.endswith(".crt") and filename != "ca.crt": + print(get_cert_display_name(os.path.join(cluster.ca_pool_dir, filename))) diff --git a/deploy/operations/certificates-management/cluster.py b/deploy/operations/certificates-management/cluster.py new file mode 100644 index 000000000..00e918aa9 --- /dev/null +++ b/deploy/operations/certificates-management/cluster.py @@ -0,0 +1,135 @@ +import os + +from utils import slugify + + +class Cluster(object): + """Represent an instance of a cluster, expose paths""" + + def __init__(self, name, cluster_context, namespace, organization, nodes_count): + self._name = name + self.cluster_context = cluster_context + self.namespace = namespace + self.organization = organization + self.nodes_count = nodes_count + + @property + def name(self): + return slugify(self._name) + + @property + def directory(self): + # Replace characters breaking folder names + def remove_special_chars(s: str): + for c in [":", "/"]: + s = s.replace(c, "_") + return s + + return os.path.join(os.getcwd(), "workspace", remove_special_chars(self._name)) + + @property + def ca_key_dir(self): + return os.path.join(self.directory, "ca") + + @property + def ca_key_file(self): + return os.path.join(self.ca_key_dir, "ca.key") + + @property + def ca_cert_file(self): + return os.path.join(self.ca_key_dir, "ca.crt") + + @property + def ca_conf(self): + return os.path.join(self.ca_key_dir, "ca.conf") + + @property + def client_certs_dir(self): + return os.path.join(self.directory, "clients") + + @property + def client_ca(self): + return os.path.join(self.client_certs_dir, "root.crt") + + @property + def master_certs_dir(self): + return os.path.join(self.directory, "masters") + + @property + def master_ca(self): + return os.path.join(self.master_certs_dir, "ca.crt") + + @property + def tserver_certs_dir(self): + return os.path.join(self.directory, "tservers") + + @property + def tserver_ca(self): + return os.path.join(self.tserver_certs_dir, "ca.crt") + + @property + def ca_pool_dir(self): + return os.path.join(self.directory, "ca_pool") + + @property + def ca_pool_ca(self): + return os.path.join(self.ca_pool_dir, "ca.crt") + + @property + def is_ready(self): + return os.path.exists(self.ca_key_file) + + @property + def clients(self): + return ["yugabytedb"] # TODO: Do we need more, like a specifc one for the DSS? + + def get_client_cert_file(self, client): + return f"{self.client_certs_dir}/{client}.crt" + + def get_client_key_file(self, client): + return f"{self.client_certs_dir}/{client}.key" + + def get_client_csr_file(self, client): + return f"{self.ca_key_dir}/client.{client}.csr" + + def get_client_conf_file(self, client): + return f"{self.ca_key_dir}/client.{client}.conf" + + def is_client_ready(self, client): + return os.path.exists(self.get_client_cert_file(client)) + + def get_node_short_name(self, node_type, node_id): + return f"yb-{node_type}-{node_id}" + + def get_node_short_name_group(self, node_type, node_id): + short_name = self.get_node_short_name(node_type, node_id) + return f"{short_name}.yb-{node_type}s" + + def get_node_full_name(self, node_type, node_id): + short_name_group = self.get_node_short_name_group(node_type, node_id) + return f"{short_name_group}.{self.namespace}.svc.cluster.local" + + def get_node_full_name_without_group(self, node_type, node_id): + short_name = self.get_node_short_name(node_type, node_id) + return f"{short_name}.{self.namespace}.svc.cluster.local" + + def get_node_cert_file(self, node_type, node_id): + folder = getattr(self, f"{node_type}_certs_dir") + full_name = self.get_node_full_name(node_type, node_id) + return f"{folder}/node.{full_name}.crt" + + def get_node_key_file(self, node_type, node_id): + folder = getattr(self, f"{node_type}_certs_dir") + full_name = self.get_node_full_name(node_type, node_id) + return f"{folder}/node.{full_name}.key" + + def get_node_csr_file(self, node_type, node_id): + full_name = self.get_node_full_name(node_type, node_id) + return f"{self.ca_key_dir}/node.{full_name}.csr" + + def get_node_conf_file(self, node_type, node_id): + full_name = self.get_node_full_name(node_type, node_id) + return f"{self.ca_key_dir}/node.{full_name}.conf" + + def is_node_ready(self, node_type, node_id): + return os.path.exists(self.get_node_cert_file(node_type, node_id)) diff --git a/deploy/operations/certificates-management/dss-certs.py b/deploy/operations/certificates-management/dss-certs.py new file mode 100755 index 000000000..5234960ff --- /dev/null +++ b/deploy/operations/certificates-management/dss-certs.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 + +import argparse +import logging +import shutil +import sys + +from apply import do_apply +from cluster import Cluster +from init import do_init +from nodes import do_generate_nodes +from ca_pool import do_get_pool_ca, do_get_ca, do_add_cas, do_list_pool_ca, do_remove_cas + +l = logging.getLogger(__name__) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Manage certificates for a yugabyte cluster" + ) + parser.add_argument( + "--name", + metavar="NAME", + required=True, + help="name of your cluster, should be unique to identify it", + ) + parser.add_argument( + "--organization", + metavar="ORGANIZATION", + default="generic-dss-organization", + help="name of your origanization", + ) + parser.add_argument( + "--cluster-context", + metavar="CLUSTER_CONTEXT", + required=True, + help="kubernetes cluster context name", + ) + parser.add_argument( + "--namespace", + metavar="NAMESPACE", + required=True, + help="kubernetes cluster namespace you are deploying to.", + ) + parser.add_argument( + "--nodes-count", + metavar="NODES_COUNT", + default="3", + help="Number of yugabyte nodes in the cluster, default to 3", + ) + parser.add_argument( + "--ca-file", + metavar="CA_FILE", + default="-", + help="CA file, for add/remove operation. Set to '-' to use stdin", + ) + parser.add_argument( + "--ca-serial", + metavar="CA_SERIAL", + help="CA serial, for remove operation. If set, --ca-file is ignored", + ) + parser.add_argument( + "action", + type=str, + help="action to be run", + choices=[ + "init", + "apply", + "regenerate-nodes", + "add-pool-ca", + "remove-pool-ca", + "list-pool-ca", + "get-pool-ca", + "get-ca", + "destroy", + ], + ) + parser.add_argument( + "--log-level", + type=str, + help="logging level", + default="INFO", + choices=[ + "DEBUG", + "INFO", + "WARNING", + "ERROR", + ], + ) + return parser.parse_args() + + +def main(): + + args = parse_args() + logging.basicConfig( + level=args.log_level, + format="%(asctime)-15s %(funcName)-25s %(levelname)-8s %(message)s", + ) + cluster = Cluster( + args.name, + args.cluster_context, + args.namespace, + args.organization, + args.nodes_count, + ) + + def read_input(): + if args.ca_file == "-": + return sys.stdin.read() + + with open(args.ca_file, 'r') as f: + return f.read() + + if args.action == "init": + do_init(cluster) + elif args.action == "regenerate-nodes": + do_generate_nodes(cluster) + elif args.action == "apply": + do_apply(cluster) + elif args.action == "add-pool-ca": + do_add_cas(cluster, read_input()) + elif args.action == "remove-pool-ca": + if args.ca_serial: + do_remove_cas(cluster, args.ca_serial) + else: + do_remove_cas(cluster, read_input()) + elif args.action == "list-pool-ca": + do_list_pool_ca(cluster) + elif args.action == "get-pool-ca": + do_get_pool_ca(cluster) + elif args.action == "get-ca": + do_get_ca(cluster) + elif args.action == "destroy": + if input("Are you sure? You will loose all your certificates! [yN]") == "y": + shutil.rmtree(cluster.directory) + l.warning(f"Destroyed cluster certificates") + else: + l.info(f"Cancelled removal") + + +if __name__ == "__main__": + main() diff --git a/deploy/operations/certificates-management/init.py b/deploy/operations/certificates-management/init.py new file mode 100644 index 000000000..675d1baca --- /dev/null +++ b/deploy/operations/certificates-management/init.py @@ -0,0 +1,240 @@ +import logging +import os +import subprocess +import sys + +from ca_pool import do_add_cas +from nodes import do_generate_nodes +from utils import get_cert_display_name + +l = logging.getLogger(__name__) + + +def generate_ca_config(cluster): + l.debug("Creating CA configuration files and database") + + with open(cluster.ca_conf, "w") as f: + f.write( + f""" + [ ca ] + default_ca = my_ca + +[ my_ca ] +default_days = 3650 + +serial = {cluster.ca_key_dir}/serial.txt +database = {cluster.ca_key_dir}/index.txt +default_md = sha256 +policy = my_policy + +[ my_policy ] + +organizationName = supplied +commonName = supplied + +[req] +prompt=no +distinguished_name = my_distinguished_name +x509_extensions = my_extensions + +[ my_distinguished_name ] +organizationName = {cluster.organization} +commonName = CA.{cluster.name} + +[ my_extensions ] +keyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment,keyCertSign +basicConstraints = critical,CA:true,pathlen:1 + +""" + ) + + with open(f"{cluster.ca_key_dir}/serial.txt", "w") as f: + f.write("0001") + + with open(f"{cluster.ca_key_dir}/index.txt", "w") as f: + f.write("") + + l.info("Created CA configuration files and database") + + +def generate_ca_key(cluster): + l.debug("Generating CA private key") + subprocess.check_call( + ["openssl", "genrsa", "-out", cluster.ca_key_file, "4096"], + stdout=subprocess.DEVNULL, + ) + l.info("Generated CA private key") + + +def generate_ca_cert(cluster): + l.debug("Generating CA certificate") + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-x509", + "-days", + "3650", + "-config", + cluster.ca_conf, + "-key", + cluster.ca_key_file, + "-out", + cluster.ca_cert_file, + ], + stdout=subprocess.DEVNULL, + ) + + name = get_cert_display_name(cluster.ca_cert_file) + + l.info(f"Generated CA certificate '{name}'") + + +def generate_ca(cluster): + generate_ca_config(cluster) + generate_ca_key(cluster) + generate_ca_cert(cluster) + + +def make_directories(cluster): + + l.debug("Creating directories") + + if not os.path.exists("workspace"): + os.makedirs("workspace") + + os.mkdir(cluster.directory) + os.mkdir(cluster.ca_key_dir) + os.mkdir(cluster.master_certs_dir) + os.mkdir(cluster.tserver_certs_dir) + os.mkdir(cluster.client_certs_dir) + os.mkdir(cluster.ca_pool_dir) + + l.info("Created directories") + + +def generate_clients(cluster): + + for client in cluster.clients: + if cluster.is_client_ready(client): + l.debug(f"Client '{client}' certificates already generated") + continue + generate_client_config(cluster, client) + generate_client_key(cluster, client) + generate_client_csr(cluster, client) + generate_client_cert(cluster, client) + + +def generate_client_config(cluster, client): + + l.debug(f"Creating client '{client}' configuration file") + + with open(cluster.get_client_conf_file(client), "w") as f: + f.write( + f"""[ req ] +prompt=no +distinguished_name = my_distinguished_name + +[ my_distinguished_name ] +organizationName = {cluster.organization} +commonName = client.{client} +""" + ) + + l.info(f"Created client '{client}' configuration file") + + +def generate_client_key(cluster, client): + + l.debug(f"Generating client '{client}' private key") + + subprocess.check_call( + ["openssl", "genrsa", "-out", cluster.get_client_key_file(client), "4096"] + ) + + l.info(f"Generated client '{client}' private key") + + +def generate_client_csr(cluster, client): + + l.debug(f"Generating client '{client}' certificate request") + + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-config", + cluster.get_client_conf_file(client), + "-key", + cluster.get_client_key_file(client), + "-out", + cluster.get_client_csr_file(client), + ], + stdout=subprocess.DEVNULL, + ) + + l.info(f"Generated client '{client}' certificate request") + + +def generate_client_cert(cluster, client): + + l.debug(f"Generating client '{client}' certificate") + + subprocess.check_call( + [ + "openssl", + "ca", + "-config", + cluster.ca_conf, + "-keyfile", + cluster.ca_key_file, + "-cert", + cluster.ca_cert_file, + "-policy", + "my_policy", + "-out", + cluster.get_client_cert_file(client), + "-outdir", + cluster.client_certs_dir, + "-in", + cluster.get_client_csr_file(client), + "-days", + "3650", + "-batch", + "-extfile", + cluster.get_client_conf_file(client), + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + name = get_cert_display_name(cluster.get_client_cert_file(client)) + + l.info(f"Generated client '{client}' certificate '{name}'") + + +def do_init(cluster): + """Initialize a new cluster""" + + l.info("Initialization of a new cluster") + + if cluster.is_ready: + l.error("Cluster is already initialized, unable to continue") + sys.exit(1) + else: + l.debug("Cluster is not already initialized, continuing") + + make_directories(cluster) + generate_ca(cluster) + generate_clients(cluster) + + do_generate_nodes(cluster) + + with open(cluster.ca_cert_file, "r") as f: + do_add_cas(cluster, f.read()) + + l.info( + "The new cluster certificates are ready! Don't forget to 'apply' the configuration." + ) diff --git a/deploy/operations/certificates-management/nodes.py b/deploy/operations/certificates-management/nodes.py new file mode 100644 index 000000000..fddf9c46d --- /dev/null +++ b/deploy/operations/certificates-management/nodes.py @@ -0,0 +1,152 @@ +import logging +import subprocess +import sys + +from utils import get_cert_display_name + + +def generate_node_config(cluster, node_type, node_id): + + l.debug(f"Creating {node_type} #{node_id} configuration file") + + short_name = cluster.get_node_short_name(node_type, node_id) + short_name_group = cluster.get_node_short_name_group(node_type, node_id) + full_name = cluster.get_node_full_name(node_type, node_id) + full_name_without_group = cluster.get_node_full_name_without_group( + node_type, node_id + ) + + with open(cluster.get_node_conf_file(node_type, node_id), "w") as f: + f.write( + f"""[ req ] +prompt=no +distinguished_name = my_distinguished_name + +[ my_distinguished_name ] +organizationName = {cluster.organization} +commonName = {full_name} + +# Multiple subject alternative names (SANs) such as IP Address, +# DNS Name, Email, URI, and so on, can be specified under this section +[ req_ext] +subjectAltName = @alt_names +[alt_names] +DNS.1 = {short_name} +DNS.2 = {full_name} +DNS.3 = {short_name_group} +DNS.4 = {full_name_without_group} +DNS.5 = yb-{node_type}s +DNS.6 = yb-{node_type}s.{cluster.namespace} +DNS.7 = yb-{node_type}s.{cluster.namespace}.svc.cluster.local +""" + ) + + l.info(f"Created {node_type} #{node_id} configuration file") + + +def generate_node_key(cluster, node_type, node_id): + + l.debug(f"Generating {node_type} #{node_id} private key") + + subprocess.check_call( + [ + "openssl", + "genrsa", + "-out", + cluster.get_node_key_file(node_type, node_id), + "4096", + ] + ) + + l.info(f"Generated {node_type} #{node_id} private key") + + +def generate_node_csr(cluster, node_type, node_id): + + l.debug(f"Generating {node_type} #{node_id} certificate request") + + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-config", + cluster.get_node_conf_file(node_type, node_id), + "-key", + cluster.get_node_key_file(node_type, node_id), + "-out", + cluster.get_node_csr_file(node_type, node_id), + ], + stdout=subprocess.DEVNULL, + ) + + l.info(f"Generated {node_type} #{node_id} certificate request") + + +def generate_node_cert(cluster, node_type, node_id): + + l.debug(f"Generating {node_type} #{node_id} certificate") + + subprocess.check_call( + [ + "openssl", + "ca", + "-config", + cluster.ca_conf, + "-keyfile", + cluster.ca_key_file, + "-cert", + cluster.ca_cert_file, + "-policy", + "my_policy", + "-out", + cluster.get_node_cert_file(node_type, node_id), + "-outdir", + getattr(cluster, f"{node_type}_certs_dir"), + "-in", + cluster.get_node_csr_file(node_type, node_id), + "-days", + "3650", + "-batch", + "-extfile", + cluster.get_node_conf_file(node_type, node_id), + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + name = get_cert_display_name(cluster.get_node_cert_file(node_type, node_id)) + + l.info(f"Generated {node_type} #{node_id} certificate '{name}'") + + +def generate_node(cluster, node_type, node_id): + if cluster.is_node_ready(node_type, node_id): + l.debug(f"{node_type} #{node_id} certificiates already generated") + return + + generate_node_config(cluster, node_type, node_id) + generate_node_key(cluster, node_type, node_id) + generate_node_csr(cluster, node_type, node_id) + generate_node_cert(cluster, node_type, node_id) + + +l = logging.getLogger(__name__) + + +def do_generate_nodes(cluster): + """Generate certificates for all nodes (master and tserver)""" + + l.info("Generation of nodes certificates") + + if not cluster.is_ready: + l.error("Cluster is not already initialized, unable to continue") + sys.exit(1) + else: + l.debug("Cluster is initialized, continuing") + + for node_type in ["master", "tserver"]: + for node_id in range(0, int(cluster.nodes_count)): + generate_node(cluster, node_type, node_id) + + l.info("All nodes certificates are ready") diff --git a/deploy/operations/certificates-management/utils.py b/deploy/operations/certificates-management/utils.py new file mode 100644 index 000000000..4947a3d3f --- /dev/null +++ b/deploy/operations/certificates-management/utils.py @@ -0,0 +1,51 @@ +import logging +import re +import ssl +import sys +import unicodedata + +l = logging.getLogger(__name__) + + +def slugify(text): + text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode("ascii") + text = text.lower() + text = re.sub(r"[^a-z0-9_\.]+", "-", text) + text = text.strip("-") + return text + + +def get_cert_display_name(path): + try: + cert_dict = ssl._ssl._test_decode_cert( + path + ) # We do use an internal function, to avoid installing dependencies + except Exception as e: + l.error(e) + sys.exit(1) + + serial = cert_dict.get("serialNumber", "") + + orga = "" + cn = "" + + for kv in cert_dict.get("subject", []): + for k, v in kv: + if k == "organizationName": + orga = v + elif k == "commonName": + cn = v + + return f"SN={serial[-8:]}, O={orga}, CN={cn}" + + +def get_cert_serial(path): + try: + cert_dict = ssl._ssl._test_decode_cert( + path + ) # We do use an internal function, to avoid installing dependencies + except Exception as e: + l.error(e) + sys.exit(1) + + return cert_dict["serialNumber"] From 85333d3c7514dbac4ce2334bea1df095bf5cc6f5 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Thu, 1 May 2025 16:07:59 +0200 Subject: [PATCH 12/18] [helm] Add TLS support on yugabyte --- .gitignore | 3 +- build/apply-certs-yugabyte.sh | 54 +++ build/make-certs-yugabyte.py | 364 ++++++++++++++++++ build/make-certs.py | 2 - .../helm-charts/dss/templates/_helpers.tpl | 2 +- .../helm-charts/dss/templates/_volumes.tpl | 22 +- .../dss/templates/dss-core-service.yaml | 15 +- .../dss/templates/schema-manager.yaml | 10 +- deploy/services/helm-charts/dss/values.yaml | 23 ++ 9 files changed, 477 insertions(+), 18 deletions(-) create mode 100755 build/apply-certs-yugabyte.sh create mode 100755 build/make-certs-yugabyte.py diff --git a/.gitignore b/.gitignore index a24e88084..689976da1 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ test_result # Generated configs build/generated/ build/workspace/ +build/workspace-yugabyte/ build/cockroachdb.yaml build/values.yaml build/dss.yaml @@ -131,4 +132,4 @@ go .vscode # terraform -.terraform* \ No newline at end of file +.terraform* diff --git a/build/apply-certs-yugabyte.sh b/build/apply-certs-yugabyte.sh new file mode 100755 index 000000000..8bcb72cb4 --- /dev/null +++ b/build/apply-certs-yugabyte.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [ "$1" == "" ]; then + echo "usage: ./apply-certs.sh " + exit 1 +fi + +if [ "$2" == "" ]; then + echo "usage: ./apply-certs.sh " + exit 1 +fi + +echo '=========================================================================' +echo '= Note that errors below are acceptable as long as the terminal message =' +echo '= is success. =' +echo '=========================================================================' + +set -e +set -x + +# Paths to directories in which to store certificates and generated YAML files. +CONTEXT="$1" +DIR="$(pwd)" +NAMESPACE="$2" + +# Replace characters breaking folder names +WORKSPACE=$(echo "${CONTEXT}" | tr ':/' '_') +CLIENTS_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/client_certs_dir" +MASTER_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/master_certs_dir" +TSERVER_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/tserver_certs_dir" +# CA_KEY_DIR="$DIR/workspace/$WORKSPACE/ca_key_dir" +# CA_CRT_DIR="$DIR/workspace/$WORKSPACE/ca_certs_dir" +JWT_PUBLIC_CERTS_DIR="$DIR/jwt-public-certs" + +# Delete previous secrets in case they have changed. +kubectl create namespace "$NAMESPACE" --context "$CONTEXT" || true + +kubectl delete secret yb-master-yugabyte-tls-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true +kubectl delete secret yb-tserver-yugabyte-tls-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true +kubectl delete secret yugabyte-tls-client-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true +kubectl delete secret dss.public.certs --namespace "$NAMESPACE" --context "$CONTEXT" || true + +kubectl create secret generic yb-master-yugabyte-tls-cert --namespace "$NAMESPACE" --from-file "$MASTER_CERTS_DIR" --context "$CONTEXT" +kubectl create secret generic yb-tserver-yugabyte-tls-cert --namespace "$NAMESPACE" --from-file "$TSERVER_CERTS_DIR" --context "$CONTEXT" +kubectl create secret generic yugabyte-tls-client-cert --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" + + +kubectl create secret generic dss.public.certs --namespace "$NAMESPACE" --from-file "$JWT_PUBLIC_CERTS_DIR" --context "$CONTEXT" + +echo '=========================================================================' +echo '= Secrets uploaded successfully. =' +echo '=========================================================================' diff --git a/build/make-certs-yugabyte.py b/build/make-certs-yugabyte.py new file mode 100755 index 000000000..3f209999e --- /dev/null +++ b/build/make-certs-yugabyte.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 + +import argparse +import os +import shutil +import subprocess + + +class YugabyteCluster(object): + + def __init__(self, cluster_context, namespace, ca_cert_to_join=None): + self._ca_cert_to_join = ca_cert_to_join + self._cluster_context = cluster_context + self._namespace = namespace + + @property + def ca_cert_to_join(self): + return self._ca_cert_to_join + + @property + def namespace(self): + return self._namespace + + @property + def directory(self): + # Replace characters breaking folder names + def remove_special_chars(s: str): + for c in [":", "/"]: + s = s.replace(c, "_") + return s + + return os.path.join( + os.getcwd(), + "workspace-yugabyte", remove_special_chars(self._cluster_context) + ) + + @property + def ca_key_dir(self): + return os.path.join(self.directory, "ca_key_dir") + + @property + def ca_key_file(self): + return os.path.join(self.ca_key_dir, "ca.key") + + @property + def ca_cert_file(self): + return os.path.join(self.ca_key_dir, "ca.crt") + + @property + def ca_conf(self): + return os.path.join(self.ca_key_dir, "ca.conf") + + @property + def ca_certs_dir(self): + return os.path.join(self.directory, "ca_certs_dir") + + @property + def client_certs_dir(self): + return os.path.join(self.directory, "client_certs_dir") + + @property + def master_certs_dir(self): + return os.path.join(self.directory, "master_certs_dir") + + @property + def tserver_certs_dir(self): + return os.path.join(self.directory, "tserver_certs_dir") + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Creates certificates for a new Cockroachdb cluster" + ) + parser.add_argument( + "--cluster-context", + metavar="CLUSTER_CONTEXT", + required=True, + help="kubernetes cluster context name", + ) + parser.add_argument( + "--namespace", + metavar="NAMESPACE", + required=True, + help="kubernetes cluster namespace you are deploying to.", + ) + # TODO + # parser.add_argument( + # "--node-address", + # metavar="ADDRESS", + # nargs="*", + # default=[], + # help="extra addresses to add to the node certificate", + # ) + parser.add_argument( + "--ca-cert-to-join", + metavar="FILENAME", + help="file containing an existing CA cert of a cluster to join.", + ) + parser.add_argument( + "--overwrite-ca-cert", + action="store_true", + default=False, + help="True to generate new CA certs, false to use the existing one", + ) + parser.add_argument( + "--node-count", + metavar="NODE_COUNT", + default="3", + help="Number of yugabyte nodes in the cluster, default to 3", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + cr = YugabyteCluster(args.cluster_context, args.namespace, args.ca_cert_to_join) + + # Create the generated directories. + if not os.path.exists("workspace-yugabyte"): + os.makedirs("workspace-yugabyte") + + if not os.path.exists(cr.directory): + os.makedirs(cr.directory) + + create_ca = not os.path.exists(cr.ca_key_dir) or args.overwrite_ca_cert + if create_ca: + # Create a new CA. + # Delete and recreate the ca_certs_dir. + shutil.rmtree(cr.ca_key_dir, ignore_errors=True) + os.mkdir(cr.ca_key_dir) + + # Build master, tserver and client certs. + os.mkdir(cr.master_certs_dir) + os.mkdir(cr.tserver_certs_dir) + os.mkdir(cr.client_certs_dir) + + if create_ca: + + with open(cr.ca_conf, "w") as f: + f.write( + f""" + [ ca ] + default_ca = my_ca + +[ my_ca ] +default_days = 3650 + +serial = {cr.ca_key_dir}/serial.txt +database = {cr.ca_key_dir}/index.txt +default_md = sha256 +policy = my_policy + +[ my_policy ] + +organizationName = supplied +commonName = supplied + +[req] +prompt=no +distinguished_name = my_distinguished_name +x509_extensions = my_extensions + +[ my_distinguished_name ] +organizationName = Yugabyte +commonName = CA for YugabyteDB + +[ my_extensions ] +keyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment,keyCertSign +basicConstraints = critical,CA:true,pathlen:1 + +""" + ) + + with open(f"{cr.ca_key_dir}/serial.txt", "w") as f: + f.write("01") + + with open(f"{cr.ca_key_dir}/index.txt", "w") as f: + f.write("") + + subprocess.check_call(["openssl", "genrsa", "-out", cr.ca_key_file]) + + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-x509", + "-days", + "3650", + "-config", + cr.ca_conf, + "-key", + cr.ca_key_file, + "-out", + cr.ca_cert_file, + ] + ) + + ### CLIENT CERTIFICATE + + # Copy CA + shutil.copy(cr.ca_cert_file, f"{cr.client_certs_dir}/root.crt") + + for client in ["yugabytedb"]: + + crt_name = f"{cr.client_certs_dir}/{client}.crt" + key_name = f"{cr.client_certs_dir}/{client}.key" + + conf_name = f"{cr.ca_key_dir}/conf.client.{client}" + csr_name = f"{cr.ca_key_dir}/csr.client.{client}" + + if os.path.exists(crt_name) and os.path.exists(key_name): # No need to regenerate it + continue + + with open(conf_name, "w") as f: + f.write( + f"""[ req ] +prompt=no +distinguished_name = my_distinguished_name + +[ my_distinguished_name ] +organizationName = Yugabyte +commonName = {client} +""" + ) + + subprocess.check_call(["openssl", "genrsa", "-out", key_name]) + + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-config", + conf_name, + "-key", + key_name, + "-out", + csr_name, + ] + ) + + subprocess.check_call( + [ + "openssl", + "ca", + "-config", + cr.ca_conf, + "-keyfile", + cr.ca_key_file, + "-cert", + cr.ca_cert_file, + "-policy", + "my_policy", + "-out", + crt_name, + "-outdir", + cr.client_certs_dir, + "-in", + csr_name, + "-days", + "3650", + "-batch", + "-extfile", + conf_name, + ] + ) + + ### SERVERS + + for server_type in ["master", "tserver"]: + + folder = getattr(cr, f"{server_type}_certs_dir") + + # Copy CA + shutil.copy(cr.ca_cert_file, folder) + + for server_id in range(0, int(args.node_count)): + short_name = f"yb-{server_type}-{server_id}" + short_name_group = f"{short_name}.yb-{server_type}s" + full_name_group = f"{short_name}.{cr.namespace}.svc.cluster.local" + full_name = f"{short_name_group}.{cr.namespace}.svc.cluster.local" + + crt_name = f"{folder}/node.{full_name}.crt" + key_name = f"{folder}/node.{full_name}.key" + + conf_name = f"{cr.ca_key_dir}/conf.{full_name}" + csr_name = f"{cr.ca_key_dir}/csr.{full_name}" + + if os.path.exists(crt_name) and os.path.exists(key_name): # No need to regenerate it + continue + + with open(conf_name, "w") as f: + f.write( + f"""[ req ] +prompt=no +distinguished_name = my_distinguished_name + +[ my_distinguished_name ] +organizationName = Yugabyte +commonName = {full_name} + +# Multiple subject alternative names (SANs) such as IP Address, +# DNS Name, Email, URI, and so on, can be specified under this section +[ req_ext] +subjectAltName = @alt_names +[alt_names] +DNS.1 = {short_name} +DNS.2 = {full_name} +DNS.3 = {short_name_group} +DNS.4 = {full_name_group} +DNS.5 = yb-{server_type}s +DNS.6 = yb-{server_type}s.{cr.namespace} +DNS.7 = yb-{server_type}s.{cr.namespace}.svc.cluster.local +""" + ) + + subprocess.check_call(["openssl", "genrsa", "-out", key_name]) + + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-config", + conf_name, + "-key", + key_name, + "-out", + csr_name, + ] + ) + + subprocess.check_call( + [ + "openssl", + "ca", + "-config", + cr.ca_conf, + "-keyfile", + cr.ca_key_file, + "-cert", + cr.ca_cert_file, + "-policy", + "my_policy", + "-out", + crt_name, + "-outdir", + folder, + "-in", + csr_name, + "-days", + "3650", + "-batch", + "-extfile", + conf_name, + "-extensions", + "req_ext", + ] + ) + + +if __name__ == "__main__": + main() diff --git a/build/make-certs.py b/build/make-certs.py index 2ea8f1b0e..1f54b8310 100755 --- a/build/make-certs.py +++ b/build/make-certs.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 import argparse -import itertools -import glob import os import shutil import subprocess diff --git a/deploy/services/helm-charts/dss/templates/_helpers.tpl b/deploy/services/helm-charts/dss/templates/_helpers.tpl index f2e569003..31ddbcc5c 100644 --- a/deploy/services/helm-charts/dss/templates/_helpers.tpl +++ b/deploy/services/helm-charts/dss/templates/_helpers.tpl @@ -50,6 +50,6 @@ yugabyte {{ if .cockroachdbEnabled }} - "/cockroach/cockroach sql --certs-dir /cockroach/cockroach-certs/ --host {{.datastoreHost}} --port \"{{.datastorePort}}\" --format raw -e \"SELECT * FROM crdb_internal.databases where name = '{{.schemaName}}';\" | grep {{.schemaName}}" {{ else }} - - "ysqlsh --host {{.datastoreHost}} --port \"{{.datastorePort}}\" -c \"SELECT datname FROM pg_database where datname = '{{.schemaName}}';\" | grep {{.schemaName}}" + - "ysqlsh --host {{.datastoreHost}} --port \"{{.datastorePort}}\" \"sslmode=require\" -c \"SELECT datname FROM pg_database where datname = '{{.schemaName}}';\" | grep {{.schemaName}}" {{ end }} {{- end -}} diff --git a/deploy/services/helm-charts/dss/templates/_volumes.tpl b/deploy/services/helm-charts/dss/templates/_volumes.tpl index e21dc30e4..e5af35405 100644 --- a/deploy/services/helm-charts/dss/templates/_volumes.tpl +++ b/deploy/services/helm-charts/dss/templates/_volumes.tpl @@ -2,27 +2,41 @@ - name: ca-certs secret: defaultMode: 256 - secretName: cockroachdb.ca.crt + secretName: {{ if .cockroachdb }}cockroachdb.ca.crt{{ else }}yugabyte-tls-client-cert{{ end }} {{- end -}} {{- define "ca-certs:volumeMount" -}} +{{ if .cockroachdb }} - mountPath: /cockroach/cockroach-certs/ca.crt name: ca-certs subPath: ca.crt +{{ else }} +- mountPath: /opt/yugabyte-certs/ca.crt + name: ca-certs + subPath: root.crt +{{- end -}} {{- end -}} - {{- define "client-certs:volume" -}} - name: client-certs secret: defaultMode: 256 - secretName: cockroachdb.client.root + secretName: {{ if .cockroachdb }}cockroachdb.client.root{{ else }}yugabyte-tls-client-cert{{ end }} {{- end -}} {{- define "client-certs:volumeMount" -}} +{{ if .cockroachdb }} - mountPath: /cockroach/cockroach-certs/client.root.crt name: client-certs subPath: client.root.crt - mountPath: /cockroach/cockroach-certs/client.root.key name: client-certs subPath: client.root.key +{{ else }} +- mountPath: /opt/yugabyte-certs/client.yugabyte.crt + name: client-certs + subPath: yugabytedb.crt +- mountPath: /opt/yugabyte-certs/client.yugabyte.key + name: client-certs + subPath: yugabytedb.key +{{- end -}} {{- end -}} @@ -35,4 +49,4 @@ {{- define "public-certs:volumeMount" -}} - mountPath: /public-certs name: public-certs -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index d129031ed..74ea05500 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -51,6 +51,9 @@ spec: - --cockroach_ssl_dir=/cockroach/cockroach-certs - --cockroach_ssl_mode=verify-full - --locality={{ .Values.cockroachdb.conf.locality }} +{{ else }} + - --cockroach_ssl_dir=/opt/yugabyte-certs + - --cockroach_ssl_mode=verify-full {{ end }} - --dump_requests=true - --enable_scd={{$dss.enableScd | default true}} @@ -80,12 +83,12 @@ spec: stdin: false tty: false volumeMounts: - {{- include "ca-certs:volumeMount" . | nindent 12 }} - {{- include "client-certs:volumeMount" . | nindent 12 }} - {{- include "public-certs:volumeMount" . | nindent 12 }} + {{- include "ca-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "client-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "public-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} imagePullSecrets: [] terminationGracePeriodSeconds: 30 volumes: - {{- include "ca-certs:volume" . | nindent 8 }} - {{- include "client-certs:volume" . | nindent 8 }} - {{- include "public-certs:volume" . | nindent 8 }} + {{- include "ca-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "client-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "public-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} diff --git a/deploy/services/helm-charts/dss/templates/schema-manager.yaml b/deploy/services/helm-charts/dss/templates/schema-manager.yaml index fbd9b22a6..6376723e0 100644 --- a/deploy/services/helm-charts/dss/templates/schema-manager.yaml +++ b/deploy/services/helm-charts/dss/templates/schema-manager.yaml @@ -46,6 +46,8 @@ spec: - --schemas_dir=/db-schemas/{{$service}} {{ else }} - --schemas_dir=/db-schemas/yugabyte/{{$service}} + - --cockroach_ssl_dir=/opt/yugabyte-certs/ + - --cockroach_ssl_mode=verify-full {{ end }} - --db_version={{$schemaVersion}} command: @@ -57,12 +59,12 @@ spec: stdin: false tty: false volumeMounts: - {{- include "ca-certs:volumeMount" . | nindent 12 }} - {{- include "client-certs:volumeMount" . | nindent 12 }} + {{- include "ca-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "client-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} imagePullSecrets: [] restartPolicy: OnFailure terminationGracePeriodSeconds: 30 volumes: - {{- include "ca-certs:volume" . | nindent 8 }} - {{- include "client-certs:volume" . | nindent 8 }} + {{- include "ca-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "client-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} {{- end -}} diff --git a/deploy/services/helm-charts/dss/values.yaml b/deploy/services/helm-charts/dss/values.yaml index dfb2ed0e2..0d4727efd 100644 --- a/deploy/services/helm-charts/dss/values.yaml +++ b/deploy/services/helm-charts/dss/values.yaml @@ -19,4 +19,27 @@ yugabyte: Image: repository: yugabytedb/yugabyte + isMultiAz: true + # multicluster: + # createServicePerPod: true + + gflags: + master: + placement_cloud: "dss" + placement_region: "uss-1" + placement_zone: "zone" + tserver: + placement_cloud: "dss" + placement_region: "uss-1" + placement_zone: "zone" + + masterAddresses: "yb-master-0.yb-masters.default.svc.cluster.local:7100,yb-master-1.yb-masters.default.svc.cluster.local:7100,yb-master-2.yb-masters.default.svc.cluster.local:7100" + + tls: + enabled: true + nodeToNode: true + clientToServer: true + insecure: false + provided: true + # See https://github.com/yugabyte/charts/blob/master/stable/yugabyte/values.yaml From b3065ba2afdfbd6e712818ec9a58024233961ed7 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 6 May 2025 14:40:42 +0200 Subject: [PATCH 13/18] Normalize cockroachdbEnabled from previous PR --- .../services/helm-charts/dss/templates/_volumes.tpl | 8 ++++---- .../helm-charts/dss/templates/dss-core-service.yaml | 12 ++++++------ .../helm-charts/dss/templates/schema-manager.yaml | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/deploy/services/helm-charts/dss/templates/_volumes.tpl b/deploy/services/helm-charts/dss/templates/_volumes.tpl index e5af35405..981e5d80c 100644 --- a/deploy/services/helm-charts/dss/templates/_volumes.tpl +++ b/deploy/services/helm-charts/dss/templates/_volumes.tpl @@ -2,10 +2,10 @@ - name: ca-certs secret: defaultMode: 256 - secretName: {{ if .cockroachdb }}cockroachdb.ca.crt{{ else }}yugabyte-tls-client-cert{{ end }} + secretName: {{ if .cockroachdbEnabled }}cockroachdb.ca.crt{{ else }}yugabyte-tls-client-cert{{ end }} {{- end -}} {{- define "ca-certs:volumeMount" -}} -{{ if .cockroachdb }} +{{ if .cockroachdbEnabled }} - mountPath: /cockroach/cockroach-certs/ca.crt name: ca-certs subPath: ca.crt @@ -19,10 +19,10 @@ - name: client-certs secret: defaultMode: 256 - secretName: {{ if .cockroachdb }}cockroachdb.client.root{{ else }}yugabyte-tls-client-cert{{ end }} + secretName: {{ if .cockroachdbEnabled }}cockroachdb.client.root{{ else }}yugabyte-tls-client-cert{{ end }} {{- end -}} {{- define "client-certs:volumeMount" -}} -{{ if .cockroachdb }} +{{ if .cockroachdbEnabled }} - mountPath: /cockroach/cockroach-certs/client.root.crt name: client-certs subPath: client.root.crt diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 74ea05500..d5c8a1913 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -83,12 +83,12 @@ spec: stdin: false tty: false volumeMounts: - {{- include "ca-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} - {{- include "client-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} - {{- include "public-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "ca-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "client-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "public-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} imagePullSecrets: [] terminationGracePeriodSeconds: 30 volumes: - {{- include "ca-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} - {{- include "client-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} - {{- include "public-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "ca-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "client-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "public-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} diff --git a/deploy/services/helm-charts/dss/templates/schema-manager.yaml b/deploy/services/helm-charts/dss/templates/schema-manager.yaml index 6376723e0..f5a232d76 100644 --- a/deploy/services/helm-charts/dss/templates/schema-manager.yaml +++ b/deploy/services/helm-charts/dss/templates/schema-manager.yaml @@ -59,12 +59,12 @@ spec: stdin: false tty: false volumeMounts: - {{- include "ca-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} - {{- include "client-certs:volumeMount" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "ca-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} + {{- include "client-certs:volumeMount" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }} imagePullSecrets: [] restartPolicy: OnFailure terminationGracePeriodSeconds: 30 volumes: - {{- include "ca-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} - {{- include "client-certs:volume" (dict "cockroachdb" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "ca-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} + {{- include "client-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }} {{- end -}} From 41813fe446f5bc5a317f1774fbf079d0da2c792c Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 13 May 2025 13:24:34 +0200 Subject: [PATCH 14/18] Update deploy/operations/certificates-management/README.md --- deploy/operations/certificates-management/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/operations/certificates-management/README.md b/deploy/operations/certificates-management/README.md index 09a0bdf7c..983698c1e 100644 --- a/deploy/operations/certificates-management/README.md +++ b/deploy/operations/certificates-management/README.md @@ -94,7 +94,7 @@ Examples: ### `remove-pool-ca` Remove the CA certificate(s) of another(s) USS in the pool of trusted certificates. -Unknown certificates are not remove again. +Unknown certificates are not removed again. You can set the file with certificate(s) with `--ca-file`, use stdin or use `--ca-serial` to specify the serial / name of the certificate you want to remove. From 7eda8dffa3063044be9a2d88be117aae60a5bec3 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 13 May 2025 13:25:31 +0200 Subject: [PATCH 15/18] Secret -> secret_name --- deploy/operations/certificates-management/apply.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/deploy/operations/certificates-management/apply.py b/deploy/operations/certificates-management/apply.py index 37234d1b8..ff056da0c 100644 --- a/deploy/operations/certificates-management/apply.py +++ b/deploy/operations/certificates-management/apply.py @@ -22,21 +22,21 @@ def do_apply(cluster): except subprocess.CalledProcessError: # We do assume everything else works l.debug(f"Namespace {cluster.namespace} already exists") - for secret in ["yb-master-yugabyte-tls-cert", "yb-tserver-yugabyte-tls-cert", "yugabyte-tls-client-cert", "dss.public.certs"]: + for secret_name in ["yb-master-yugabyte-tls-cert", "yb-tserver-yugabyte-tls-cert", "yugabyte-tls-client-cert", "dss.public.certs"]: try: subprocess.check_call( - ["kubectl", "delete", "secret", secret, "--namespace", cluster.namespace, "--context", cluster.cluster_context], + ["kubectl", "delete", "secret", secret_name, "--namespace", cluster.namespace, "--context", cluster.cluster_context], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) - l.info(f"Deleted old secret '{secret}'") + l.info(f"Deleted old secret '{secret_name}'") except subprocess.CalledProcessError: # We do assume everything else works - l.debug(f"Secret '{secret}' not present on the cluster") + l.debug(f"Secret '{secret_name}' not present on the cluster") - for secret, folder in [ + for secret_name, folder in [ ("yb-master-yugabyte-tls-cert", cluster.master_certs_dir), ("yb-tserver-yugabyte-tls-cert", cluster.tserver_certs_dir), ("yugabyte-tls-client-cert", cluster.client_certs_dir), @@ -44,9 +44,9 @@ def do_apply(cluster): ]: subprocess.check_call( - ["kubectl", "create", "secret", "generic", secret, "--namespace", cluster.namespace, "--context", cluster.cluster_context, "--from-file", folder], + ["kubectl", "create", "secret", "generic", secret_name, "--namespace", cluster.namespace, "--context", cluster.cluster_context, "--from-file", folder], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) - l.info(f"Created secret '{secret}'") + l.info(f"Created secret '{secret_name}'") From 5ccbe66a476af6075b144627d3e3163420d652cd Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 27 May 2025 11:30:54 +0200 Subject: [PATCH 16/18] Yugabyte in GCP --- .../terraform-aws-kubernetes/variables.gen.tf | 17 +++ .../terraform-commons-dss/helm.tf | 132 +++++++++++++++++- .../terraform-commons-dss/scripts.tf | 12 ++ .../templates/dss-certs.sh.tmp | 17 +++ .../terraform-commons-dss/variables.gen.tf | 76 ++++++++++ .../variables_internal.tf | 16 +++ .../terraform-google-kubernetes/cluster.tf | 21 ++- .../terraform-google-kubernetes/dns.tf | 24 +++- .../terraform-google-kubernetes/output.tf | 26 ++++ .../variables.gen.tf | 17 +++ .../modules/terraform-aws-dss/TFVARS.gen.md | 42 ++++++ .../terraform-aws-dss/variables.gen.tf | 76 ++++++++++ .../terraform-google-dss/TFVARS.gen.md | 42 ++++++ .../modules/terraform-google-dss/main.tf | 47 ++++--- .../modules/terraform-google-dss/output.tf | 8 ++ .../terraform.dev.example.tfvars | 6 + .../terraform-google-dss/variables.gen.tf | 76 ++++++++++ .../utils/definitions/datastore_type.tf | 15 ++ .../utils/definitions/yugabyte_cloud.tf | 10 ++ .../definitions/yugabyte_external_nodes.tf | 9 ++ .../definitions/yugabyte_light_resources.tf | 10 ++ .../utils/definitions/yugabyte_region.tf | 10 ++ .../utils/definitions/yugabyte_resources.tf | 11 ++ .../utils/definitions/yugabyte_zone.tf | 10 ++ deploy/infrastructure/utils/variables.py | 7 +- .../certificates-management/cluster.py | 18 ++- .../certificates-management/dss-certs.py | 7 + .../certificates-management/nodes.py | 30 +++- deploy/operations/ci/aws-1/variables.gen.tf | 76 ++++++++++ .../dss/templates/dss-core-service.yaml | 2 +- .../dss/templates/yugabyte-loadbalancers.yaml | 92 ++++++++++++ 31 files changed, 931 insertions(+), 31 deletions(-) create mode 100644 deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp create mode 100644 deploy/infrastructure/utils/definitions/datastore_type.tf create mode 100644 deploy/infrastructure/utils/definitions/yugabyte_cloud.tf create mode 100644 deploy/infrastructure/utils/definitions/yugabyte_external_nodes.tf create mode 100644 deploy/infrastructure/utils/definitions/yugabyte_light_resources.tf create mode 100644 deploy/infrastructure/utils/definitions/yugabyte_region.tf create mode 100644 deploy/infrastructure/utils/definitions/yugabyte_resources.tf create mode 100644 deploy/infrastructure/utils/definitions/yugabyte_zone.tf create mode 100644 deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf index 3ec55966b..12cf2ba1a 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf @@ -65,6 +65,23 @@ variable "crdb_hostname_suffix" { EOT } +variable "datastore_type" { + type = string + description = <<-EOT + Type of datastore used + + Supported technologies: cockroachdb, yugabyte + EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" +} + + variable "cluster_name" { type = string description = <<-EOT diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf index fc5b73ec0..cb38910c2 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf @@ -6,7 +6,7 @@ locals { resource "local_file" "helm_chart_values" { filename = "${local.workspace_location}/helm_values.yml" - content = yamlencode({ + content = var.datastore_type == "cockroachdb" ? yamlencode({ cockroachdb = { image = { tag = var.crdb_image_tag @@ -69,5 +69,133 @@ resource "local_file" "helm_chart_values" { global = { cloudProvider = var.kubernetes_cloud_provider_name } - }) +}) : yamlencode({ + cockroachdb = { + enabled = false + image = { + tag = "dummy" + } + fullnameOverride = "dummy" + conf = { + cluster-name = "dummy" + locality = "dummy" + } + statefulset = {} + } + yugabyte = { + enabled = true + Image = { + tag = "2.25.2.0-b359" + } + nameOverride = "dss-yugabyte" + + resource = var.yugabyte_light_resources ? { + master = { + requests = { + cpu = "0.1" + memory = "0.5G" + } + } + tserver = { + requests = { + cpu = "0.1" + memory = "0.5G" + } + } + } : {} + enableLoadBalancer = false + + master = { + extraEnv = [{ + name = "HOSTNAMENO" + valueFrom = { + fieldRef = { + fieldPath = "metadata.labels['apps.kubernetes.io/pod-index']" + } + } + }] + serverBroadcastAddress: "$${HOSTNAMENO}.master.${var.crdb_hostname_suffix}" + rpcBindAddress: "$${HOSTNAMENO}.master.${var.crdb_hostname_suffix}" + preCommands: "sed -E \"/\\.svc\\.cluster\\.local/ s/^([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)([[:space:]]+)/\\1 $(echo \"$${HOSTNAMENO}.master.${var.crdb_hostname_suffix}\" | sed 's/[\\/&]/\\\\&/g')\\2/\" /etc/hosts > /tmp/newhosts && /bin/cp /tmp/newhosts /etc/hosts && \\" + } + + tserver = { + extraEnv = [{ + name = "HOSTNAMENO" + valueFrom = { + fieldRef = { + fieldPath = "metadata.labels['apps.kubernetes.io/pod-index']" + } + } + }] + serverBroadcastAddress: "$${HOSTNAMENO}.tserver.${var.crdb_hostname_suffix}" + rpcBindAddress: "$${HOSTNAMENO}.tserver.${var.crdb_hostname_suffix}" + preCommands: "sed -E \"/\\.svc\\.cluster\\.local/ s/^([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)([[:space:]]+)/\\1 $(echo \"$${HOSTNAMENO}.tserver.${var.crdb_hostname_suffix}\" | sed 's/[\\/&]/\\\\&/g')\\2/\" /etc/hosts > /tmp/newhosts && /bin/cp /tmp/newhosts /etc/hosts && \\" + } + + gflags = { + master = { + placement_cloud: var.yugabyte_cloud + placement_region: var.yugabyte_region + placement_zone: var.yugabyte_zone + use_private_ip: "zone" + } + tserver = { + placement_cloud: var.yugabyte_cloud + placement_region: var.yugabyte_region + placement_zone: var.yugabyte_zone + use_private_ip: "zone" + } + } + + isMultiAz = true + masterAddresses = join(",", ["0.master.${var.crdb_hostname_suffix},1.master.${var.crdb_hostname_suffix},2.master.${var.crdb_hostname_suffix}", join(",", var.yugabyte_external_nodes)]) + } + + loadBalancers = { + cockroachdbNodes = [] + + yugabyteMasterNodes = [ + for ip in var.yugabyte_internal_masters_nodes[*].ip : + { + ip = ip + subnet = var.workload_subnet + } + ] + + yugabyteTserverNodes = [ + for ip in var.yugabyte_internal_tservers_nodes[*].ip : + { + ip = ip + subnet = var.workload_subnet + } + ] + + dssGateway = { + ip = var.ip_gateway + subnet = var.workload_subnet + certName = var.gateway_cert_name + sslPolicy = var.ssl_policy + } + } + + dss = { + image = var.image + + conf = { + pubKeys = [ + "/test-certs/auth2.pem" + ] + jwksEndpoint = var.authorization.jwks != null ? var.authorization.jwks.endpoint : "" + jwksKeyIds = var.authorization.jwks != null ? [var.authorization.jwks.key_id] : [] + hostname = var.app_hostname + enableScd = var.enable_scd + } + } + + global = { + cloudProvider = var.kubernetes_cloud_provider_name + } +}) + } diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf index 01ee71f17..7f2981d40 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf @@ -1,5 +1,6 @@ resource "local_file" "make_certs" { + count = var.datastore_type == "cockroachdb" ? 1 : 0 content = templatefile("${path.module}/templates/make-certs.sh.tmp", { cluster_context = var.kubernetes_context_name namespace = var.kubernetes_namespace @@ -10,6 +11,7 @@ resource "local_file" "make_certs" { } resource "local_file" "apply_certs" { + count = var.datastore_type == "cockroachdb" ? 1 : 0 content = templatefile("${path.module}/templates/apply-certs.sh.tmp", { cluster_context = var.kubernetes_context_name namespace = var.kubernetes_namespace @@ -17,6 +19,16 @@ resource "local_file" "apply_certs" { filename = "${local.workspace_location}/apply-certs.sh" } +resource "local_file" "dss_certs" { + count = var.datastore_type == "yugabyte" ? 1 : 0 + content = templatefile("${path.module}/templates/dss-certs.sh.tmp", { + cluster_context = var.kubernetes_context_name + namespace = var.kubernetes_namespace + crdb_hostname_suffix = var.crdb_hostname_suffix + }) + filename = "${local.workspace_location}/dss-certs.sh" +} + resource "local_file" "get_credentials" { content = templatefile("${path.module}/templates/get-credentials.sh.tmp", { get_credentials_cmd = var.kubernetes_get_credentials_cmd diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp new file mode 100644 index 000000000..8ee81325b --- /dev/null +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# This file was automatically generated by terraform-commons-dss. +# Do not edit it directly. + +set -eo pipefail + +OS=$(uname) +if [[ "$OS" == "Darwin" ]]; then + # OSX uses BSD readlink + BASEDIR="$(dirname "$0")" +else + BASEDIR=$(readlink -e "$(dirname "$0")") +fi +cd "$BASEDIR/../../../deploy/operations/certificates-management/" || exit 1 + +./dss-certs.py --name ${cluster_context} --organization default_orga --cluster-context ${cluster_context} --nodes-public-address "..${crdb_hostname_suffix}" --namespace ${namespace} "$@" diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf index 32be949d2..2cfdc68bf 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf @@ -22,6 +22,23 @@ variable "crdb_hostname_suffix" { EOT } +variable "datastore_type" { + type = string + description = <<-EOT + Type of datastore used + + Supported technologies: cockroachdb, yugabyte + EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" +} + + variable "image" { type = string description = <<-EOT @@ -225,3 +242,62 @@ variable "kubernetes_namespace" { } } +variable "yugabyte_cloud" { + type = string + description = <<-EOT + Cloud of yugabyte instances, used for partionning. + + Should be set to dss unless you're doing advanced partitionning. + EOT + + default = "dss" +} + + +variable "yugabyte_region" { + type = string + description = <<-EOT + Region of yugabyte instances, used for partionning. + + Should be different from others USS in a cluster. + EOT + + default = "uss-1" +} + + +variable "yugabyte_zone" { + type = string + description = <<-EOT + Zone of yugabyte instances, used for partionning. + + Should be set to zone unless you're doing advanced partitionning. + EOT + + default = "zone" +} + + +variable "yugabyte_light_resources" { + type = bool + description = <<-EOT + Enable light resources reservation for yugabyte instances. + + Useful for a dev cluster when you don't want to overload your kubernetes cluster. + EOT + + default = false +} + + +variable "yugabyte_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. + Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"] + EOT + default = [] +} + + + diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables_internal.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables_internal.tf index 0c6b28b3e..d56ca25f1 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables_internal.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables_internal.tf @@ -29,6 +29,22 @@ variable "crdb_internal_nodes" { description = "List of the IP addresses and related dns for the Cockroach DB nodes" } +variable "yugabyte_internal_masters_nodes" { + type = list(object({ + dns = string + ip = string + })) + description = "List of the IP addresses and related dns for the Yugabyte DB master nodes" +} + +variable "yugabyte_internal_tservers_nodes" { + type = list(object({ + dns = string + ip = string + })) + description = "List of the IP addresses and related dns for the Yugabyte DB tserver nodes" +} + variable "ip_gateway" { type = string description = "IP of the gateway used by the DSS service" diff --git a/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf b/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf index cb0416c01..4a6c925fa 100644 --- a/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf +++ b/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf @@ -52,7 +52,7 @@ resource "google_compute_global_address" "ip_gateway" { # Static IP addresses for CRDB instances resource "google_compute_address" "ip_crdb" { - count = var.node_count + count = var.datastore_type == "cockroachdb" ? var.node_count : 0 name = format("%s-ip-crdb%v", var.cluster_name, count.index) region = local.region @@ -60,6 +60,25 @@ resource "google_compute_address" "ip_crdb" { description = format("%s.%s", count.index, var.crdb_hostname_suffix) } +# Static IP addresses for yugabyte instances +resource "google_compute_address" "ip_yugabyte_masters" { + count = var.datastore_type == "yugabyte" ? var.node_count : 0 + name = format("%s-ip-yugabyte-master%v", var.cluster_name, count.index) + region = local.region + + # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames. + description = format("%s.master.%s", count.index, var.crdb_hostname_suffix) +} + +resource "google_compute_address" "ip_yugabyte_tservers" { + count = var.datastore_type == "yugabyte" ? var.node_count : 0 + name = format("%s-ip-yugabyte-tserver%v", var.cluster_name, count.index) + region = local.region + + # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames. + description = format("%s.tserver.%s", count.index, var.crdb_hostname_suffix) +} + locals { kubectl_cluster_context_name = format("gke_%s_%s_%s", google_container_cluster.kubernetes_cluster.project, google_container_cluster.kubernetes_cluster.location, google_container_cluster.kubernetes_cluster.name) } diff --git a/deploy/infrastructure/dependencies/terraform-google-kubernetes/dns.tf b/deploy/infrastructure/dependencies/terraform-google-kubernetes/dns.tf index 85147bcc8..f35ae6227 100644 --- a/deploy/infrastructure/dependencies/terraform-google-kubernetes/dns.tf +++ b/deploy/infrastructure/dependencies/terraform-google-kubernetes/dns.tf @@ -16,11 +16,31 @@ resource "google_dns_record_set" "gateway" { } resource "google_dns_record_set" "crdb" { - count = var.google_dns_managed_zone_name == "" ? 0 : var.node_count + count = var.google_dns_managed_zone_name == "" || var.datastore_type != "cockroachdb" ? 0 : var.node_count name = "${google_compute_address.ip_crdb[count.index].description}." # description contains the expected hostname type = "A" ttl = 300 managed_zone = data.google_dns_managed_zone.default[0].name rrdatas = [google_compute_address.ip_crdb[count.index].address] -} \ No newline at end of file +} + +resource "google_dns_record_set" "yugabyte_masters" { + count = var.google_dns_managed_zone_name == "" || var.datastore_type != "yugabyte" ? 0 : var.node_count + name = "${google_compute_address.ip_yugabyte_masters[count.index].description}." # description contains the expected hostname + type = "A" + ttl = 300 + + managed_zone = data.google_dns_managed_zone.default[0].name + rrdatas = [google_compute_address.ip_yugabyte_masters[count.index].address] +} + +resource "google_dns_record_set" "yugabyte_tserver" { + count = var.google_dns_managed_zone_name == "" || var.datastore_type != "yugabyte" ? 0 : var.node_count + name = "${google_compute_address.ip_yugabyte_tservers[count.index].description}." # description contains the expected hostname + type = "A" + ttl = 300 + + managed_zone = data.google_dns_managed_zone.default[0].name + rrdatas = [google_compute_address.ip_yugabyte_tservers[count.index].address] +} diff --git a/deploy/infrastructure/dependencies/terraform-google-kubernetes/output.tf b/deploy/infrastructure/dependencies/terraform-google-kubernetes/output.tf index aa2a27cda..b239e572b 100644 --- a/deploy/infrastructure/dependencies/terraform-google-kubernetes/output.tf +++ b/deploy/infrastructure/dependencies/terraform-google-kubernetes/output.tf @@ -2,6 +2,14 @@ output "crdb_addresses" { value = [for a in google_compute_address.ip_crdb[*] : { expected_dns : a.description, address : a.address }] } +output "yugabyte_masters_addresses" { + value = [for a in google_compute_address.ip_yugabyte_masters[*] : { expected_dns : a.description, address : a.address }] +} + +output "yugabyte_tservers_addresses" { + value = [for a in google_compute_address.ip_yugabyte_tservers[*] : { expected_dns : a.description, address : a.address }] +} + output "gateway_address" { value = { expected_dns : google_compute_global_address.ip_gateway.description, @@ -41,3 +49,21 @@ output "crdb_nodes" { } ] } + +output "yugabyte_masters_nodes" { + value = [ + for i in google_compute_address.ip_yugabyte_masters : { + ip = i.address + dns = i.description + } + ] +} + +output "yugabyte_tservers_nodes" { + value = [ + for i in google_compute_address.ip_yugabyte_tservers : { + ip = i.address + dns = i.description + } + ] +} diff --git a/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf index 07d364c55..584adf2bb 100644 --- a/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf @@ -54,6 +54,23 @@ variable "crdb_hostname_suffix" { EOT } +variable "datastore_type" { + type = string + description = <<-EOT + Type of datastore used + + Supported technologies: cockroachdb, yugabyte + EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" +} + + variable "cluster_name" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md index dfaaf89cc..d8b6fccf5 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md @@ -63,6 +63,13 @@ Leave empty to disable record creation.

For instance, if your CRDB nodes were addressable at 0.db.example.com, 1.db.example.com and 2.db.example.com, then the value would be db.example.com.

Example: db.example.com

+ + + datastore_type + string + "cockroachdb" +

Type of datastore used

+

Supported technologies: cockroachdb, yugabyte

cluster_name @@ -243,6 +250,41 @@ Example: ["0.db.dss.example.com", "1.db.dss.example.com", &q "default"

Namespace where to deploy Kubernetes resources. Only default is supported at the moment.

Example: default

+ + + yugabyte_cloud + string + "dss" +

Cloud of yugabyte instances, used for partionning.

+

Should be set to dss unless you're doing advanced partitionning.

+ + + yugabyte_region + string + "uss-1" +

Region of yugabyte instances, used for partionning.

+

Should be different from others USS in a cluster.

+ + + yugabyte_zone + string + "zone" +

Zone of yugabyte instances, used for partionning.

+

Should be set to zone unless you're doing advanced partitionning.

+ + + yugabyte_light_resources + bool + false +

Enable light resources reservation for yugabyte instances.

+

Useful for a dev cluster when you don't want to overload your kubernetes cluster.

+ + + yugabyte_external_nodes + list(string) + [] +

Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. +Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"]

\ No newline at end of file diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf index d5bcaca07..7cb4337d7 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf @@ -65,6 +65,23 @@ variable "crdb_hostname_suffix" { EOT } +variable "datastore_type" { + type = string + description = <<-EOT + Type of datastore used + + Supported technologies: cockroachdb, yugabyte + EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" +} + + variable "cluster_name" { type = string description = <<-EOT @@ -319,3 +336,62 @@ variable "kubernetes_namespace" { } } +variable "yugabyte_cloud" { + type = string + description = <<-EOT + Cloud of yugabyte instances, used for partionning. + + Should be set to dss unless you're doing advanced partitionning. + EOT + + default = "dss" +} + + +variable "yugabyte_region" { + type = string + description = <<-EOT + Region of yugabyte instances, used for partionning. + + Should be different from others USS in a cluster. + EOT + + default = "uss-1" +} + + +variable "yugabyte_zone" { + type = string + description = <<-EOT + Zone of yugabyte instances, used for partionning. + + Should be set to zone unless you're doing advanced partitionning. + EOT + + default = "zone" +} + + +variable "yugabyte_light_resources" { + type = bool + description = <<-EOT + Enable light resources reservation for yugabyte instances. + + Useful for a dev cluster when you don't want to overload your kubernetes cluster. + EOT + + default = false +} + + +variable "yugabyte_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. + Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"] + EOT + default = [] +} + + + diff --git a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md index 07f85cd79..eeb649317 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md @@ -59,6 +59,13 @@ Example: n2-standard-4 for production, e2-medium for d For instance, if your CRDB nodes were addressable at 0.db.example.com, 1.db.example.com and 2.db.example.com, then the value would be db.example.com.

Example: db.example.com

+ + + datastore_type + string + "cockroachdb" +

Type of datastore used

+

Supported technologies: cockroachdb, yugabyte

cluster_name @@ -239,6 +246,41 @@ Example: ["0.db.dss.example.com", "1.db.dss.example.com", &q "default"

Namespace where to deploy Kubernetes resources. Only default is supported at the moment.

Example: default

+ + + yugabyte_cloud + string + "dss" +

Cloud of yugabyte instances, used for partionning.

+

Should be set to dss unless you're doing advanced partitionning.

+ + + yugabyte_region + string + "uss-1" +

Region of yugabyte instances, used for partionning.

+

Should be different from others USS in a cluster.

+ + + yugabyte_zone + string + "zone" +

Zone of yugabyte instances, used for partionning.

+

Should be set to zone unless you're doing advanced partitionning.

+ + + yugabyte_light_resources + bool + false +

Enable light resources reservation for yugabyte instances.

+

Useful for a dev cluster when you don't want to overload your kubernetes cluster.

+ + + yugabyte_external_nodes + list(string) + [] +

Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. +Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"]

\ No newline at end of file diff --git a/deploy/infrastructure/modules/terraform-google-dss/main.tf b/deploy/infrastructure/modules/terraform-google-dss/main.tf index ea98a4dd6..8b2782dd4 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/main.tf @@ -5,6 +5,7 @@ module "terraform-google-kubernetes" { google_zone = var.google_zone app_hostname = var.app_hostname crdb_hostname_suffix = var.crdb_hostname_suffix + datastore_type = var.datastore_type google_dns_managed_zone_name = var.google_dns_managed_zone_name google_machine_type = var.google_machine_type node_count = var.node_count @@ -15,25 +16,33 @@ module "terraform-google-kubernetes" { module "terraform-commons-dss" { # See variables.tf for variables description. - image = var.image - kubernetes_namespace = var.kubernetes_namespace - kubernetes_storage_class = var.google_kubernetes_storage_class - app_hostname = var.app_hostname - crdb_image_tag = var.crdb_image_tag - crdb_cluster_name = var.crdb_cluster_name - crdb_hostname_suffix = var.crdb_hostname_suffix - should_init = var.should_init - authorization = var.authorization - crdb_locality = var.crdb_locality - image_pull_secret = var.image_pull_secret - crdb_external_nodes = var.crdb_external_nodes - kubernetes_api_endpoint = module.terraform-google-kubernetes.kubernetes_api_endpoint - crdb_internal_nodes = module.terraform-google-kubernetes.crdb_nodes - ip_gateway = module.terraform-google-kubernetes.ip_gateway - ssl_policy = module.terraform-google-kubernetes.ssl_policy - kubernetes_cloud_provider_name = module.terraform-google-kubernetes.kubernetes_cloud_provider_name - kubernetes_context_name = module.terraform-google-kubernetes.kubernetes_context_name - kubernetes_get_credentials_cmd = module.terraform-google-kubernetes.kubernetes_get_credentials_cmd + image = var.image + kubernetes_namespace = var.kubernetes_namespace + kubernetes_storage_class = var.google_kubernetes_storage_class + app_hostname = var.app_hostname + crdb_image_tag = var.crdb_image_tag + crdb_cluster_name = var.crdb_cluster_name + crdb_hostname_suffix = var.crdb_hostname_suffix + datastore_type = var.datastore_type + should_init = var.should_init + authorization = var.authorization + crdb_locality = var.crdb_locality + image_pull_secret = var.image_pull_secret + crdb_external_nodes = var.crdb_external_nodes + yugabyte_cloud = var.yugabyte_cloud + yugabyte_region = var.yugabyte_region + yugabyte_zone = var.yugabyte_zone + yugabyte_light_resources = var.yugabyte_light_resources + yugabyte_external_nodes = var.yugabyte_external_nodes + kubernetes_api_endpoint = module.terraform-google-kubernetes.kubernetes_api_endpoint + crdb_internal_nodes = module.terraform-google-kubernetes.crdb_nodes + yugabyte_internal_masters_nodes = module.terraform-google-kubernetes.yugabyte_masters_nodes + yugabyte_internal_tservers_nodes = module.terraform-google-kubernetes.yugabyte_tservers_nodes + ip_gateway = module.terraform-google-kubernetes.ip_gateway + ssl_policy = module.terraform-google-kubernetes.ssl_policy + kubernetes_cloud_provider_name = module.terraform-google-kubernetes.kubernetes_cloud_provider_name + kubernetes_context_name = module.terraform-google-kubernetes.kubernetes_context_name + kubernetes_get_credentials_cmd = module.terraform-google-kubernetes.kubernetes_get_credentials_cmd source = "../../dependencies/terraform-commons-dss" } diff --git a/deploy/infrastructure/modules/terraform-google-dss/output.tf b/deploy/infrastructure/modules/terraform-google-dss/output.tf index 7be13880f..0f1486a3d 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/output.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/output.tf @@ -2,6 +2,14 @@ output "crdb_addresses" { value = module.terraform-google-kubernetes.crdb_addresses } +output "yugabyte_masters_addresses" { + value = module.terraform-google-kubernetes.yugabyte_masters_addresses +} + +output "yugabyte_tservers_addresses" { + value = module.terraform-google-kubernetes.yugabyte_tservers_addresses +} + output "gateway_address" { value = module.terraform-google-kubernetes.gateway_address } diff --git a/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars b/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars index c5808f9cd..3a22909ca 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars +++ b/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars @@ -26,8 +26,14 @@ authorization = { } should_init = true +# Datastore +datastore_type = "cockroachdb" + # CockroachDB crdb_image_tag = "v24.1.3" crdb_cluster_name = "interuss_example" crdb_locality = "interuss_dss-dev-w6a" crdb_external_nodes = [] + +# Yugabyte +yugabyte_region = "uss-1" diff --git a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf index 52337cb51..86d54c973 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf @@ -54,6 +54,23 @@ variable "crdb_hostname_suffix" { EOT } +variable "datastore_type" { + type = string + description = <<-EOT + Type of datastore used + + Supported technologies: cockroachdb, yugabyte + EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" +} + + variable "cluster_name" { type = string description = <<-EOT @@ -308,3 +325,62 @@ variable "kubernetes_namespace" { } } +variable "yugabyte_cloud" { + type = string + description = <<-EOT + Cloud of yugabyte instances, used for partionning. + + Should be set to dss unless you're doing advanced partitionning. + EOT + + default = "dss" +} + + +variable "yugabyte_region" { + type = string + description = <<-EOT + Region of yugabyte instances, used for partionning. + + Should be different from others USS in a cluster. + EOT + + default = "uss-1" +} + + +variable "yugabyte_zone" { + type = string + description = <<-EOT + Zone of yugabyte instances, used for partionning. + + Should be set to zone unless you're doing advanced partitionning. + EOT + + default = "zone" +} + + +variable "yugabyte_light_resources" { + type = bool + description = <<-EOT + Enable light resources reservation for yugabyte instances. + + Useful for a dev cluster when you don't want to overload your kubernetes cluster. + EOT + + default = false +} + + +variable "yugabyte_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. + Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"] + EOT + default = [] +} + + + diff --git a/deploy/infrastructure/utils/definitions/datastore_type.tf b/deploy/infrastructure/utils/definitions/datastore_type.tf new file mode 100644 index 000000000..b63a17237 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/datastore_type.tf @@ -0,0 +1,15 @@ +variable "datastore_type" { + type = string + description = <<-EOT + Type of datastore used + + Supported technologies: cockroachdb, yugabyte + EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" +} diff --git a/deploy/infrastructure/utils/definitions/yugabyte_cloud.tf b/deploy/infrastructure/utils/definitions/yugabyte_cloud.tf new file mode 100644 index 000000000..51ef755d6 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_cloud.tf @@ -0,0 +1,10 @@ +variable "yugabyte_cloud" { + type = string + description = <<-EOT + Cloud of yugabyte instances, used for partionning. + + Should be set to dss unless you're doing advanced partitionning. + EOT + + default = "dss" +} diff --git a/deploy/infrastructure/utils/definitions/yugabyte_external_nodes.tf b/deploy/infrastructure/utils/definitions/yugabyte_external_nodes.tf new file mode 100644 index 000000000..fd065346d --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_external_nodes.tf @@ -0,0 +1,9 @@ +variable "yugabyte_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. + Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"] + EOT + default = [] +} + diff --git a/deploy/infrastructure/utils/definitions/yugabyte_light_resources.tf b/deploy/infrastructure/utils/definitions/yugabyte_light_resources.tf new file mode 100644 index 000000000..f776f7a79 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_light_resources.tf @@ -0,0 +1,10 @@ +variable "yugabyte_light_resources" { + type = bool + description = <<-EOT + Enable light resources reservation for yugabyte instances. + + Useful for a dev cluster when you don't want to overload your kubernetes cluster. + EOT + + default = false +} diff --git a/deploy/infrastructure/utils/definitions/yugabyte_region.tf b/deploy/infrastructure/utils/definitions/yugabyte_region.tf new file mode 100644 index 000000000..17613b9d1 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_region.tf @@ -0,0 +1,10 @@ +variable "yugabyte_region" { + type = string + description = <<-EOT + Region of yugabyte instances, used for partionning. + + Should be different from others USS in a cluster. + EOT + + default = "uss-1" +} diff --git a/deploy/infrastructure/utils/definitions/yugabyte_resources.tf b/deploy/infrastructure/utils/definitions/yugabyte_resources.tf new file mode 100644 index 000000000..abd429c92 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_resources.tf @@ -0,0 +1,11 @@ +variable "should_init" { + type = bool + description = <<-EOT + Set to false if joining an existing pool, true if creating the first DSS instance + for a pool. When set true, this can initialize the data directories on your cluster, + and prevent you from joining an existing pool. + + Example: `true` + EOT +} + diff --git a/deploy/infrastructure/utils/definitions/yugabyte_zone.tf b/deploy/infrastructure/utils/definitions/yugabyte_zone.tf new file mode 100644 index 000000000..735133f14 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_zone.tf @@ -0,0 +1,10 @@ +variable "yugabyte_zone" { + type = string + description = <<-EOT + Zone of yugabyte instances, used for partionning. + + Should be set to zone unless you're doing advanced partitionning. + EOT + + default = "zone" +} diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 779720041..732c867c1 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -23,7 +23,7 @@ # Variables per project # For all */terraform-* -GLOBAL_VARIABLES = ["app_hostname", "crdb_hostname_suffix"] +GLOBAL_VARIABLES = ["app_hostname", "crdb_hostname_suffix", "datastore_type"] # dependencies/terraform-commons-dss COMMONS_DSS_VARIABLES = GLOBAL_VARIABLES + [ @@ -39,6 +39,11 @@ "crdb_locality", "crdb_external_nodes", "kubernetes_namespace", + "yugabyte_cloud", + "yugabyte_region", + "yugabyte_zone", + "yugabyte_light_resources", + "yugabyte_external_nodes", ] # dependencies/terraform-*-kubernetes diff --git a/deploy/operations/certificates-management/cluster.py b/deploy/operations/certificates-management/cluster.py index 00e918aa9..c65d376be 100644 --- a/deploy/operations/certificates-management/cluster.py +++ b/deploy/operations/certificates-management/cluster.py @@ -6,12 +6,13 @@ class Cluster(object): """Represent an instance of a cluster, expose paths""" - def __init__(self, name, cluster_context, namespace, organization, nodes_count): + def __init__(self, name, cluster_context, namespace, organization, nodes_count, nodes_public_address): self._name = name self.cluster_context = cluster_context self.namespace = namespace self.organization = organization self.nodes_count = nodes_count + self.nodes_public_address = nodes_public_address @property def name(self): @@ -113,6 +114,9 @@ def get_node_full_name_without_group(self, node_type, node_id): short_name = self.get_node_short_name(node_type, node_id) return f"{short_name}.{self.namespace}.svc.cluster.local" + def get_node_public_address(self, node_type, node_id): + return self.nodes_public_address.replace("", str(node_id)).replace("", node_type) + def get_node_cert_file(self, node_type, node_id): folder = getattr(self, f"{node_type}_certs_dir") full_name = self.get_node_full_name(node_type, node_id) @@ -123,6 +127,18 @@ def get_node_key_file(self, node_type, node_id): full_name = self.get_node_full_name(node_type, node_id) return f"{folder}/node.{full_name}.key" + def get_node_cert_second_file(self, node_type, node_id): + folder = getattr(self, f"{node_type}_certs_dir") + address = self.get_node_public_address(node_type, node_id) + if address: + return f"{folder}/node.{address}.crt" + + def get_node_key_second_file(self, node_type, node_id): + folder = getattr(self, f"{node_type}_certs_dir") + address = self.get_node_public_address(node_type, node_id) + if address: + return f"{folder}/node.{address}.key" + def get_node_csr_file(self, node_type, node_id): full_name = self.get_node_full_name(node_type, node_id) return f"{self.ca_key_dir}/node.{full_name}.csr" diff --git a/deploy/operations/certificates-management/dss-certs.py b/deploy/operations/certificates-management/dss-certs.py index 5234960ff..0cfbe821e 100755 --- a/deploy/operations/certificates-management/dss-certs.py +++ b/deploy/operations/certificates-management/dss-certs.py @@ -48,6 +48,12 @@ def parse_args(): default="3", help="Number of yugabyte nodes in the cluster, default to 3", ) + parser.add_argument( + "--nodes-public-address", + metavar="NODES_PUBLIC_ADDRESS", + default="", + help="Public node address. Use to indicate id of the node (0, 1, ...), for the type (tserver, masters). Example: '..db.interuss.example'", + ) parser.add_argument( "--ca-file", metavar="CA_FILE", @@ -103,6 +109,7 @@ def main(): args.namespace, args.organization, args.nodes_count, + args.nodes_public_address, ) def read_input(): diff --git a/deploy/operations/certificates-management/nodes.py b/deploy/operations/certificates-management/nodes.py index fddf9c46d..87799c7e4 100644 --- a/deploy/operations/certificates-management/nodes.py +++ b/deploy/operations/certificates-management/nodes.py @@ -1,6 +1,7 @@ import logging import subprocess import sys +import shutil from utils import get_cert_display_name @@ -15,6 +16,7 @@ def generate_node_config(cluster, node_type, node_id): full_name_without_group = cluster.get_node_full_name_without_group( node_type, node_id ) + public_address = cluster.get_node_public_address(node_type, node_id) with open(cluster.get_node_conf_file(node_type, node_id), "w") as f: f.write( @@ -28,9 +30,9 @@ def generate_node_config(cluster, node_type, node_id): # Multiple subject alternative names (SANs) such as IP Address, # DNS Name, Email, URI, and so on, can be specified under this section -[ req_ext] +[ req_ext ] subjectAltName = @alt_names -[alt_names] +[ alt_names ] DNS.1 = {short_name} DNS.2 = {full_name} DNS.3 = {short_name_group} @@ -41,6 +43,10 @@ def generate_node_config(cluster, node_type, node_id): """ ) + if public_address: + f.write(f"""DNS.8 = {public_address} +""") + l.info(f"Created {node_type} #{node_id} configuration file") @@ -48,16 +54,23 @@ def generate_node_key(cluster, node_type, node_id): l.debug(f"Generating {node_type} #{node_id} private key") + file = cluster.get_node_key_file(node_type, node_id) + subprocess.check_call( [ "openssl", "genrsa", "-out", - cluster.get_node_key_file(node_type, node_id), + file, "4096", ] ) + second_file = cluster.get_node_key_second_file(node_type, node_id) + + if second_file: + shutil.copy(file, second_file) + l.info(f"Generated {node_type} #{node_id} private key") @@ -87,6 +100,8 @@ def generate_node_cert(cluster, node_type, node_id): l.debug(f"Generating {node_type} #{node_id} certificate") + file = cluster.get_node_cert_file(node_type, node_id) + subprocess.check_call( [ "openssl", @@ -100,7 +115,7 @@ def generate_node_cert(cluster, node_type, node_id): "-policy", "my_policy", "-out", - cluster.get_node_cert_file(node_type, node_id), + file, "-outdir", getattr(cluster, f"{node_type}_certs_dir"), "-in", @@ -110,11 +125,18 @@ def generate_node_cert(cluster, node_type, node_id): "-batch", "-extfile", cluster.get_node_conf_file(node_type, node_id), + "-extensions", + "req_ext", ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) + second_file = cluster.get_node_cert_second_file(node_type, node_id) + + if second_file: + shutil.copy(file, second_file) + name = get_cert_display_name(cluster.get_node_cert_file(node_type, node_id)) l.info(f"Generated {node_type} #{node_id} certificate '{name}'") diff --git a/deploy/operations/ci/aws-1/variables.gen.tf b/deploy/operations/ci/aws-1/variables.gen.tf index d5bcaca07..7cb4337d7 100644 --- a/deploy/operations/ci/aws-1/variables.gen.tf +++ b/deploy/operations/ci/aws-1/variables.gen.tf @@ -65,6 +65,23 @@ variable "crdb_hostname_suffix" { EOT } +variable "datastore_type" { + type = string + description = <<-EOT + Type of datastore used + + Supported technologies: cockroachdb, yugabyte + EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" +} + + variable "cluster_name" { type = string description = <<-EOT @@ -319,3 +336,62 @@ variable "kubernetes_namespace" { } } +variable "yugabyte_cloud" { + type = string + description = <<-EOT + Cloud of yugabyte instances, used for partionning. + + Should be set to dss unless you're doing advanced partitionning. + EOT + + default = "dss" +} + + +variable "yugabyte_region" { + type = string + description = <<-EOT + Region of yugabyte instances, used for partionning. + + Should be different from others USS in a cluster. + EOT + + default = "uss-1" +} + + +variable "yugabyte_zone" { + type = string + description = <<-EOT + Zone of yugabyte instances, used for partionning. + + Should be set to zone unless you're doing advanced partitionning. + EOT + + default = "zone" +} + + +variable "yugabyte_light_resources" { + type = bool + description = <<-EOT + Enable light resources reservation for yugabyte instances. + + Useful for a dev cluster when you don't want to overload your kubernetes cluster. + EOT + + default = false +} + + +variable "yugabyte_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. + Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"] + EOT + default = [] +} + + + diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index d5c8a1913..6521a0fbf 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -22,7 +22,7 @@ metadata: name: {{.Release.Name}}-core-service spec: minReadySeconds: 30 - replicas: {{ len .Values.loadBalancers.cockroachdbNodes }} + replicas: {{ if $.Values.cockroachdb.enabled -}}{{ len .Values.loadBalancers.cockroachdbNodes }}{{- else -}}{{ len .Values.loadBalancers.yugabyteMasterNodes }}{{- end }} selector: matchLabels: app: {{.Release.Name}}-core-service diff --git a/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml b/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml new file mode 100644 index 000000000..f3b47ce21 --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml @@ -0,0 +1,92 @@ +{{- $cloudProvider := $.Values.global.cloudProvider}} + +{{- if $.Values.yugabyte.enabled }} + +# Master nodes Gateways +{{- range $i, $lb := .Values.loadBalancers.yugabyteMasterNodes }} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + {{- include (printf "%s-lb-crdb-annotations" $cloudProvider) + (dict + "name" (printf "%s-%s" "yugabyte-db-master-external-node" ( $i | toString) ) + "ip" $lb.ip + "subnet" $lb.subnet + "cloudProvider" $cloudProvider + ) | nindent 4 + }} + labels: + app: yugabyte + name: yugabyte-db-master-external-node-{{$i}} + name: yugabyte-db-master-external-node-{{$i}} +spec: + {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}} + ports: + - name: yugabyte-master-db-external-node-{{$i}} + port: 7100 + targetPort: 7100 + - name: yugabyte-master-ui-external-node-{{$i}} + port: 7000 + targetPort: 7000 + - name: yugabyte-master-ui2-external-node-{{$i}} + port: 9000 + targetPort: 9000 + publishNotReadyAddresses: true + selector: + statefulset.kubernetes.io/pod-name: yb-master-{{$i}} + type: LoadBalancer +{{- end }} + +# Tserver nodes Gateways +{{- range $i, $lb := .Values.loadBalancers.yugabyteTserverNodes }} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + {{- include (printf "%s-lb-crdb-annotations" $cloudProvider) + (dict + "name" (printf "%s-%s" " yugabyte-db-tserver-external-node" ( $i | toString) ) + "ip" $lb.ip + "subnet" $lb.subnet + "cloudProvider" $cloudProvider + ) | nindent 4 + }} + labels: + app: yugabyte + name: yugabyte-db-tserver-external-node-{{$i}} + name: yugabyte-db-tserver-external-node-{{$i}} +spec: + {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}} + ports: + - name: yugabyte-tserver-db-external-node-{{$i}} + port: 9100 + targetPort: 9100 + - name: yugabyte-tserver-ui-external-node-{{$i}} + port: 9000 + targetPort: 9000 + - name: yugabyte-tserver-ui2-external-node-{{$i}} + port: 7000 + targetPort: 7000 + - name: yugabyte-tserver-ycql-external-node-{{$i}} + port: 9042 + targetPort: 9042 + - name: yugabyte-tserver-ysql-external-node-{{$i}} + port: 5433 + targetPort: 5433 + - name: yugabyte-tserver-metrics-external-node-{{$i}} + port: 13000 + targetPort: 13000 + - name: yugabyte-tserver-metrics-2-external-node-{{$i}} + port: 12000 + targetPort: 12000 + publishNotReadyAddresses: true + selector: + statefulset.kubernetes.io/pod-name: yb-tserver-{{$i}} + type: LoadBalancer +{{- end }} +{{- end }} From e470cbe00c876bd8201b994d2f264503808386a0 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 27 May 2025 15:49:22 +0200 Subject: [PATCH 17/18] Yugaybte in AWS --- .../terraform-aws-kubernetes/network_dns.tf | 24 +++++++++ .../terraform-aws-kubernetes/network_lb.tf | 26 +++++++++- .../terraform-aws-kubernetes/output.tf | 34 ++++++++++++- .../terraform-commons-dss/helm.tf | 13 +++++ .../modules/terraform-aws-dss/main.tf | 49 +++++++++++-------- .../modules/terraform-aws-dss/output.tf | 8 +++ .../dss/templates/yugabyte-loadbalancers.yaml | 32 ++++++------ 7 files changed, 148 insertions(+), 38 deletions(-) diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_dns.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_dns.tf index 355cb8f99..77e439a3a 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_dns.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_dns.tf @@ -1,6 +1,8 @@ locals { crdb_hostnames = var.aws_route53_zone_id == "" ? {} : { for i in aws_eip.ip_crdb[*] : i.tags.ExpectedDNS => i.public_ip } + yugabyte_master_hostnames = var.aws_route53_zone_id == "" ? {} : { for i in aws_eip.ip_yugabyte_masters[*] : i.tags.ExpectedDNS => i.public_ip } + yugabyte_tserver_hostnames = var.aws_route53_zone_id == "" ? {} : { for i in aws_eip.ip_yugabyte_tservers[*] : i.tags.ExpectedDNS => i.public_ip } } @@ -37,3 +39,25 @@ resource "aws_route53_record" "crdb_hostname" { ttl = 300 records = [each.value] } + +# Yugabyte master nodes DNS +resource "aws_route53_record" "yugabyte_master_hostnames" { + for_each = local.yugabyte_master_hostnames + + zone_id = var.aws_route53_zone_id + name = each.key + type = "A" + ttl = 300 + records = [each.value] +} + +# Yugabyte tserver nodes DNS +resource "aws_route53_record" "yugabyte_tserver_hostnames" { + for_each = local.yugabyte_tserver_hostnames + + zone_id = var.aws_route53_zone_id + name = each.key + type = "A" + ttl = 300 + records = [each.value] +} diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf index b436a49c7..1ccd8bcda 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf @@ -73,7 +73,7 @@ resource "aws_eip" "gateway" { # Public Elastic IPs for the crdb instances resource "aws_eip" "ip_crdb" { - count = var.node_count + count = var.datastore_type == "cockroachdb" ? var.node_count : 0 vpc = true tags = { @@ -82,3 +82,27 @@ resource "aws_eip" "ip_crdb" { ExpectedDNS = format("%s.%s", count.index, var.crdb_hostname_suffix) } } + +# Public Elastic IPs for the yubagybte master instances +resource "aws_eip" "ip_yugabyte_masters" { + count = var.datastore_type == "yugabyte" ? var.node_count : 0 + vpc = true + + tags = { + Name = format("%s-ip-yugabyte-master%v", var.cluster_name, count.index) + # Preserve mapping between ips and hostnames + ExpectedDNS = format("%s.master.%s", count.index, var.crdb_hostname_suffix) + } +} + +# Public Elastic IPs for the yubagybte tserver instances +resource "aws_eip" "ip_yugabyte_tservers" { + count = var.datastore_type == "yugabyte" ? var.node_count : 0 + vpc = true + + tags = { + Name = format("%s-ip-yugabyte-tserver%v", var.cluster_name, count.index) + # Preserve mapping between ips and hostnames + ExpectedDNS = format("%s.tserver.%s", count.index, var.crdb_hostname_suffix) + } +} diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf index a6d238d77..e7bf3dc25 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf @@ -30,10 +30,42 @@ output "crdb_nodes" { ] } +output "yugabyte_masters_nodes" { + value = [ + for i in aws_eip.ip_yugabyte_masters : { + ip = i.allocation_id + dns = i.tags.ExpectedDNS + } + ] + depends_on = [ + aws_eip.ip_yugabyte_masters + ] +} + +output "yugabyte_tservers_nodes" { + value = [ + for i in aws_eip.ip_yugabyte_tservers : { + ip = i.allocation_id + dns = i.tags.ExpectedDNS + } + ] + depends_on = [ + aws_eip.ip_yugabyte_tservers + ] +} + output "crdb_addresses" { value = [for i in aws_eip.ip_crdb[*] : { expected_dns : i.tags.ExpectedDNS, address : i.public_ip }] } +output "yugabyte_masters_addresses" { + value = [for i in aws_eip.ip_yugabyte_masters[*] : { expected_dns : i.tags.ExpectedDNS, address : i.public_ip }] +} + +output "yugabyte_tservers_addresses" { + value = [for i in aws_eip.ip_yugabyte_tservers[*] : { expected_dns : i.tags.ExpectedDNS, address : i.public_ip }] +} + output "gateway_address" { value = { expected_dns : aws_eip.gateway[0].tags.ExpectedDNS, @@ -56,4 +88,4 @@ output "workload_subnet" { output "iam_role_node_group_arn" { value = aws_iam_role.dss-cluster-node-group.arn -} \ No newline at end of file +} diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf index cb38910c2..d685caf58 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf @@ -105,6 +105,19 @@ resource "local_file" "helm_chart_values" { } : {} enableLoadBalancer = false + storage = { + master = { + storageClass = var.kubernetes_storage_class + } + tserver = { + storageClass = var.kubernetes_storage_class + } + } + + preflight = { + skipUlimit = true + } + master = { extraEnv = [{ name = "HOSTNAMENO" diff --git a/deploy/infrastructure/modules/terraform-aws-dss/main.tf b/deploy/infrastructure/modules/terraform-aws-dss/main.tf index effdfae82..9000f9ca4 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/main.tf @@ -4,6 +4,7 @@ module "terraform-aws-kubernetes" { aws_region = var.aws_region app_hostname = var.app_hostname crdb_hostname_suffix = var.crdb_hostname_suffix + datastore_type = var.datastore_type aws_instance_type = var.aws_instance_type aws_route53_zone_id = var.aws_route53_zone_id aws_iam_permissions_boundary = var.aws_iam_permissions_boundary @@ -15,26 +16,34 @@ module "terraform-aws-kubernetes" { module "terraform-commons-dss" { # See variables.tf for variables description. - image = var.image - image_pull_secret = var.image_pull_secret - kubernetes_namespace = var.kubernetes_namespace - kubernetes_storage_class = var.aws_kubernetes_storage_class - app_hostname = var.app_hostname - crdb_image_tag = var.crdb_image_tag - crdb_cluster_name = var.crdb_cluster_name - crdb_hostname_suffix = var.crdb_hostname_suffix - should_init = var.should_init - authorization = var.authorization - crdb_locality = var.crdb_locality - crdb_external_nodes = var.crdb_external_nodes - crdb_internal_nodes = module.terraform-aws-kubernetes.crdb_nodes - ip_gateway = module.terraform-aws-kubernetes.ip_gateway - kubernetes_api_endpoint = module.terraform-aws-kubernetes.kubernetes_api_endpoint - kubernetes_cloud_provider_name = module.terraform-aws-kubernetes.kubernetes_cloud_provider_name - kubernetes_context_name = module.terraform-aws-kubernetes.kubernetes_context_name - kubernetes_get_credentials_cmd = module.terraform-aws-kubernetes.kubernetes_get_credentials_cmd - workload_subnet = module.terraform-aws-kubernetes.workload_subnet - gateway_cert_name = module.terraform-aws-kubernetes.app_hostname_cert_arn + image = var.image + image_pull_secret = var.image_pull_secret + kubernetes_namespace = var.kubernetes_namespace + kubernetes_storage_class = var.aws_kubernetes_storage_class + app_hostname = var.app_hostname + crdb_image_tag = var.crdb_image_tag + crdb_cluster_name = var.crdb_cluster_name + crdb_hostname_suffix = var.crdb_hostname_suffix + datastore_type = var.datastore_type + should_init = var.should_init + authorization = var.authorization + crdb_locality = var.crdb_locality + crdb_external_nodes = var.crdb_external_nodes + yugabyte_cloud = var.yugabyte_cloud + yugabyte_region = var.yugabyte_region + yugabyte_zone = var.yugabyte_zone + yugabyte_light_resources = var.yugabyte_light_resources + yugabyte_external_nodes = var.yugabyte_external_nodes + crdb_internal_nodes = module.terraform-aws-kubernetes.crdb_nodes + yugabyte_internal_masters_nodes = module.terraform-aws-kubernetes.yugabyte_masters_nodes + yugabyte_internal_tservers_nodes = module.terraform-aws-kubernetes.yugabyte_tservers_nodes + ip_gateway = module.terraform-aws-kubernetes.ip_gateway + kubernetes_api_endpoint = module.terraform-aws-kubernetes.kubernetes_api_endpoint + kubernetes_cloud_provider_name = module.terraform-aws-kubernetes.kubernetes_cloud_provider_name + kubernetes_context_name = module.terraform-aws-kubernetes.kubernetes_context_name + kubernetes_get_credentials_cmd = module.terraform-aws-kubernetes.kubernetes_get_credentials_cmd + workload_subnet = module.terraform-aws-kubernetes.workload_subnet + gateway_cert_name = module.terraform-aws-kubernetes.app_hostname_cert_arn source = "../../dependencies/terraform-commons-dss" } diff --git a/deploy/infrastructure/modules/terraform-aws-dss/output.tf b/deploy/infrastructure/modules/terraform-aws-dss/output.tf index 4ab65ab50..9d282a99f 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/output.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/output.tf @@ -2,6 +2,14 @@ output "crdb_addresses" { value = module.terraform-aws-kubernetes.crdb_addresses } +output "yugabyte_masters_addresses" { + value = module.terraform-aws-kubernetes.yugabyte_masters_addresses +} + +output "yugabyte_tservers_addresses" { + value = module.terraform-aws-kubernetes.yugabyte_tservers_addresses +} + output "gateway_address" { value = module.terraform-aws-kubernetes.gateway_address } diff --git a/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml b/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml index f3b47ce21..5614ff32d 100644 --- a/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml +++ b/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml @@ -12,7 +12,7 @@ metadata: service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" {{- include (printf "%s-lb-crdb-annotations" $cloudProvider) (dict - "name" (printf "%s-%s" "yugabyte-db-master-external-node" ( $i | toString) ) + "name" (printf "%s-%s" "yuga-master-ext" ( $i | toString) ) "ip" $lb.ip "subnet" $lb.subnet "cloudProvider" $cloudProvider @@ -20,18 +20,18 @@ metadata: }} labels: app: yugabyte - name: yugabyte-db-master-external-node-{{$i}} - name: yugabyte-db-master-external-node-{{$i}} + name: yuga-master-ext-{{$i}} + name: yuga-master-ext-{{$i}} spec: {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}} ports: - - name: yugabyte-master-db-external-node-{{$i}} + - name: yugabyte-master-db-ext-{{$i}} port: 7100 targetPort: 7100 - - name: yugabyte-master-ui-external-node-{{$i}} + - name: yugabyte-master-ui-ext-{{$i}} port: 7000 targetPort: 7000 - - name: yugabyte-master-ui2-external-node-{{$i}} + - name: yugabyte-master-ui2-ext-{{$i}} port: 9000 targetPort: 9000 publishNotReadyAddresses: true @@ -50,7 +50,7 @@ metadata: service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" {{- include (printf "%s-lb-crdb-annotations" $cloudProvider) (dict - "name" (printf "%s-%s" " yugabyte-db-tserver-external-node" ( $i | toString) ) + "name" (printf "%s-%s" " yuga-tserver-ext" ( $i | toString) ) "ip" $lb.ip "subnet" $lb.subnet "cloudProvider" $cloudProvider @@ -58,30 +58,30 @@ metadata: }} labels: app: yugabyte - name: yugabyte-db-tserver-external-node-{{$i}} - name: yugabyte-db-tserver-external-node-{{$i}} + name: yuga-tserver-ext-{{$i}} + name: yuga-tserver-ext-{{$i}} spec: {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}} ports: - - name: yugabyte-tserver-db-external-node-{{$i}} + - name: yugabyte-tserver-db-ext-{{$i}} port: 9100 targetPort: 9100 - - name: yugabyte-tserver-ui-external-node-{{$i}} + - name: yugabyte-tserver-ui-ext-{{$i}} port: 9000 targetPort: 9000 - - name: yugabyte-tserver-ui2-external-node-{{$i}} + - name: yugabyte-tserver-ui2-ext-{{$i}} port: 7000 targetPort: 7000 - - name: yugabyte-tserver-ycql-external-node-{{$i}} + - name: yugabyte-tserver-ycql-ext-{{$i}} port: 9042 targetPort: 9042 - - name: yugabyte-tserver-ysql-external-node-{{$i}} + - name: yugabyte-tserver-ysql-ext-{{$i}} port: 5433 targetPort: 5433 - - name: yugabyte-tserver-metrics-external-node-{{$i}} + - name: yugabyte-tserver-metrics-ext-{{$i}} port: 13000 targetPort: 13000 - - name: yugabyte-tserver-metrics-2-external-node-{{$i}} + - name: yugabyte-tserver-metrics-2-ext-{{$i}} port: 12000 targetPort: 12000 publishNotReadyAddresses: true From 38bcb08922aabbe12986d5beb88f8902d505b7aa Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 27 May 2025 16:43:58 +0200 Subject: [PATCH 18/18] Yugabyte: Cleanup --- build/README.md | 38 +- build/apply-certs-yugabyte.sh | 54 --- build/make-certs-yugabyte.py | 364 ------------------ .../terraform-aws-kubernetes/README.md | 1 - .../terraform-aws-kubernetes/network_lb.tf | 6 +- .../terraform-aws-kubernetes/variables.gen.tf | 9 +- .../terraform-commons-dss/helm.tf | 16 +- .../terraform-commons-dss/scripts.tf | 2 +- .../terraform-commons-dss/tanka.tf | 2 +- .../templates/dss-certs.sh.tmp | 2 +- .../templates/main.jsonnet.tmp | 2 +- .../terraform-commons-dss/variables.gen.tf | 12 +- .../terraform-google-kubernetes/cluster.tf | 6 +- .../variables.gen.tf | 9 +- .../modules/terraform-aws-dss/README.md | 19 +- .../modules/terraform-aws-dss/TFVARS.gen.md | 9 +- .../modules/terraform-aws-dss/main.tf | 4 +- .../terraform.dev.example.tfvars | 13 +- .../terraform-aws-dss/variables.gen.tf | 12 +- .../modules/terraform-google-dss/README.md | 17 +- .../terraform-google-dss/TFVARS.gen.md | 9 +- .../modules/terraform-google-dss/main.tf | 4 +- .../terraform.dev.example.tfvars | 9 +- .../terraform-google-dss/variables.gen.tf | 12 +- .../utils/definitions/crdb_hostname_suffix.tf | 10 - .../utils/definitions/db_hostname_suffix.tf | 10 + .../utils/definitions/should_init.tf | 4 +- .../utils/definitions/yugabyte_resources.tf | 11 - deploy/infrastructure/utils/variables.py | 2 +- deploy/operations/ci/aws-1/main.tf | 2 +- deploy/operations/ci/aws-1/terraform.tfvars | 2 +- deploy/operations/ci/aws-1/variables.gen.tf | 12 +- .../tanka/examples/minimum/main.jsonnet | 2 +- 33 files changed, 158 insertions(+), 528 deletions(-) delete mode 100755 build/apply-certs-yugabyte.sh delete mode 100755 build/make-certs-yugabyte.py delete mode 100644 deploy/infrastructure/utils/definitions/crdb_hostname_suffix.tf create mode 100644 deploy/infrastructure/utils/definitions/db_hostname_suffix.tf delete mode 100644 deploy/infrastructure/utils/definitions/yugabyte_resources.tf diff --git a/build/README.md b/build/README.md index 4e8facccf..683b8365a 100644 --- a/build/README.md +++ b/build/README.md @@ -209,7 +209,7 @@ a PR to that effect would be greatly appreciated. to create DNS entries for the static IP addresses created above. To list the IP addresses, use `gcloud compute addresses list`. -1. Use [`make-certs.py` script](./make-certs.py) to create certificates for +1. (Only if you use CockroachDB) Use [`make-certs.py` script](./make-certs.py) to create certificates for the CockroachDB nodes in this DSS instance: ./make-certs.py --cluster $CLUSTER_CONTEXT --namespace $NAMESPACE @@ -243,6 +243,8 @@ a PR to that effect would be greatly appreciated. the rest of the instances, such that ca.crt is the same across all instances. +1. (Only if you use Yugabyte) Use [`css-certs.py` script](../deploy/operations/certificates-management/README.md) to create certificates for the Yugabyte nodes in this DSS instance. + 1. If joining an existing DSS pool, share ca.crt with the DSS instance(s) you are trying to join, and have them apply the new ca.crt, which now contains both your instance's and the original instance's public certs, to enable @@ -251,14 +253,28 @@ a PR to that effect would be greatly appreciated. actions below. While they are performing those actions, you may continue with the instructions. - 1. Overwrite its existing ca.crt with the new ca.crt provided by the DSS - instance joining the pool. - 1. Upload the new ca.crt to its cluster using - `./apply-certs.sh $CLUSTER_CONTEXT $NAMESPACE` - 1. Restart their CockroachDB pods to recognize the updated ca.crt: - `kubectl rollout restart statefulset/cockroachdb --namespace $NAMESPACE` - 1. Inform you when their CockroachDB pods have finished restarting - (typically around 10 minutes) + 1. If you use CockroachDB: + + 1. Overwrite its existing ca.crt with the new ca.crt provided by the DSS + instance joining the pool. + 1. Upload the new ca.crt to its cluster using + `./apply-certs.sh $CLUSTER_CONTEXT $NAMESPACE` + 1. Restart their CockroachDB pods to recognize the updated ca.crt: + `kubectl rollout restart statefulset/cockroachdb --namespace $NAMESPACE` + 1. Inform you when their CockroachDB pods have finished restarting + (typically around 10 minutes) + + 1. If you use Yugabyte + + 1. Share your CA with `./dss-certs.py get-ca` + 1. Add others CAs of the pool with `./dss-certs.py add-pool-ca` + 1. Upload the new CAs to its cluster using + `./dss-certs.py apply` + 1. Restart their Yugabyte pods to recognize the updated ca.crt: + `kubectl rollout restart statefulset/yb-master --namespace $NAMESPACE` + `kubectl rollout restart statefulset/yb-tserver --namespace $NAMESPACE` + 1. Inform you when their Yugabyte pods have finished restarting + (typically around 10 minutes) 1. Ensure the Docker images are built according to the instructions in the [previous section](#docker-images). @@ -295,10 +311,10 @@ a PR to that effect would be greatly appreciated. DSS v0.16, the recommended CockroachDB image name is `cockroachdb/cockroach:v21.2.7`. From DSS v0.17, the recommended CockroachDB version is `cockroachdb/cockroach:v24.1.3`. - 1. `VAR_CRDB_HOSTNAME_SUFFIX`: The domain name suffix shared by all of your + 1. `VAR_DB_HOSTNAME_SUFFIX`: The domain name suffix shared by all of your CockroachDB nodes. For instance, if your CRDB nodes were addressable at `0.db.example.com`, `1.db.example.com`, and `2.db.example.com`, then - VAR_CRDB_HOSTNAME_SUFFIX would be `db.example.com`. + VAR_DB_HOSTNAME_SUFFIX would be `db.example.com`. 1. `VAR_CRDB_LOCALITY`: Unique name for your DSS instance. Currently, we recommend "_", and the `=` character is not diff --git a/build/apply-certs-yugabyte.sh b/build/apply-certs-yugabyte.sh deleted file mode 100755 index 8bcb72cb4..000000000 --- a/build/apply-certs-yugabyte.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -if [ "$1" == "" ]; then - echo "usage: ./apply-certs.sh " - exit 1 -fi - -if [ "$2" == "" ]; then - echo "usage: ./apply-certs.sh " - exit 1 -fi - -echo '=========================================================================' -echo '= Note that errors below are acceptable as long as the terminal message =' -echo '= is success. =' -echo '=========================================================================' - -set -e -set -x - -# Paths to directories in which to store certificates and generated YAML files. -CONTEXT="$1" -DIR="$(pwd)" -NAMESPACE="$2" - -# Replace characters breaking folder names -WORKSPACE=$(echo "${CONTEXT}" | tr ':/' '_') -CLIENTS_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/client_certs_dir" -MASTER_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/master_certs_dir" -TSERVER_CERTS_DIR="$DIR/workspace-yugabyte/$WORKSPACE/tserver_certs_dir" -# CA_KEY_DIR="$DIR/workspace/$WORKSPACE/ca_key_dir" -# CA_CRT_DIR="$DIR/workspace/$WORKSPACE/ca_certs_dir" -JWT_PUBLIC_CERTS_DIR="$DIR/jwt-public-certs" - -# Delete previous secrets in case they have changed. -kubectl create namespace "$NAMESPACE" --context "$CONTEXT" || true - -kubectl delete secret yb-master-yugabyte-tls-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true -kubectl delete secret yb-tserver-yugabyte-tls-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true -kubectl delete secret yugabyte-tls-client-cert --namespace "$NAMESPACE" --context "$CONTEXT" || true -kubectl delete secret dss.public.certs --namespace "$NAMESPACE" --context "$CONTEXT" || true - -kubectl create secret generic yb-master-yugabyte-tls-cert --namespace "$NAMESPACE" --from-file "$MASTER_CERTS_DIR" --context "$CONTEXT" -kubectl create secret generic yb-tserver-yugabyte-tls-cert --namespace "$NAMESPACE" --from-file "$TSERVER_CERTS_DIR" --context "$CONTEXT" -kubectl create secret generic yugabyte-tls-client-cert --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" - - -kubectl create secret generic dss.public.certs --namespace "$NAMESPACE" --from-file "$JWT_PUBLIC_CERTS_DIR" --context "$CONTEXT" - -echo '=========================================================================' -echo '= Secrets uploaded successfully. =' -echo '=========================================================================' diff --git a/build/make-certs-yugabyte.py b/build/make-certs-yugabyte.py deleted file mode 100755 index 3f209999e..000000000 --- a/build/make-certs-yugabyte.py +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import shutil -import subprocess - - -class YugabyteCluster(object): - - def __init__(self, cluster_context, namespace, ca_cert_to_join=None): - self._ca_cert_to_join = ca_cert_to_join - self._cluster_context = cluster_context - self._namespace = namespace - - @property - def ca_cert_to_join(self): - return self._ca_cert_to_join - - @property - def namespace(self): - return self._namespace - - @property - def directory(self): - # Replace characters breaking folder names - def remove_special_chars(s: str): - for c in [":", "/"]: - s = s.replace(c, "_") - return s - - return os.path.join( - os.getcwd(), - "workspace-yugabyte", remove_special_chars(self._cluster_context) - ) - - @property - def ca_key_dir(self): - return os.path.join(self.directory, "ca_key_dir") - - @property - def ca_key_file(self): - return os.path.join(self.ca_key_dir, "ca.key") - - @property - def ca_cert_file(self): - return os.path.join(self.ca_key_dir, "ca.crt") - - @property - def ca_conf(self): - return os.path.join(self.ca_key_dir, "ca.conf") - - @property - def ca_certs_dir(self): - return os.path.join(self.directory, "ca_certs_dir") - - @property - def client_certs_dir(self): - return os.path.join(self.directory, "client_certs_dir") - - @property - def master_certs_dir(self): - return os.path.join(self.directory, "master_certs_dir") - - @property - def tserver_certs_dir(self): - return os.path.join(self.directory, "tserver_certs_dir") - - -def parse_args(): - parser = argparse.ArgumentParser( - description="Creates certificates for a new Cockroachdb cluster" - ) - parser.add_argument( - "--cluster-context", - metavar="CLUSTER_CONTEXT", - required=True, - help="kubernetes cluster context name", - ) - parser.add_argument( - "--namespace", - metavar="NAMESPACE", - required=True, - help="kubernetes cluster namespace you are deploying to.", - ) - # TODO - # parser.add_argument( - # "--node-address", - # metavar="ADDRESS", - # nargs="*", - # default=[], - # help="extra addresses to add to the node certificate", - # ) - parser.add_argument( - "--ca-cert-to-join", - metavar="FILENAME", - help="file containing an existing CA cert of a cluster to join.", - ) - parser.add_argument( - "--overwrite-ca-cert", - action="store_true", - default=False, - help="True to generate new CA certs, false to use the existing one", - ) - parser.add_argument( - "--node-count", - metavar="NODE_COUNT", - default="3", - help="Number of yugabyte nodes in the cluster, default to 3", - ) - return parser.parse_args() - - -def main(): - args = parse_args() - cr = YugabyteCluster(args.cluster_context, args.namespace, args.ca_cert_to_join) - - # Create the generated directories. - if not os.path.exists("workspace-yugabyte"): - os.makedirs("workspace-yugabyte") - - if not os.path.exists(cr.directory): - os.makedirs(cr.directory) - - create_ca = not os.path.exists(cr.ca_key_dir) or args.overwrite_ca_cert - if create_ca: - # Create a new CA. - # Delete and recreate the ca_certs_dir. - shutil.rmtree(cr.ca_key_dir, ignore_errors=True) - os.mkdir(cr.ca_key_dir) - - # Build master, tserver and client certs. - os.mkdir(cr.master_certs_dir) - os.mkdir(cr.tserver_certs_dir) - os.mkdir(cr.client_certs_dir) - - if create_ca: - - with open(cr.ca_conf, "w") as f: - f.write( - f""" - [ ca ] - default_ca = my_ca - -[ my_ca ] -default_days = 3650 - -serial = {cr.ca_key_dir}/serial.txt -database = {cr.ca_key_dir}/index.txt -default_md = sha256 -policy = my_policy - -[ my_policy ] - -organizationName = supplied -commonName = supplied - -[req] -prompt=no -distinguished_name = my_distinguished_name -x509_extensions = my_extensions - -[ my_distinguished_name ] -organizationName = Yugabyte -commonName = CA for YugabyteDB - -[ my_extensions ] -keyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment,keyCertSign -basicConstraints = critical,CA:true,pathlen:1 - -""" - ) - - with open(f"{cr.ca_key_dir}/serial.txt", "w") as f: - f.write("01") - - with open(f"{cr.ca_key_dir}/index.txt", "w") as f: - f.write("") - - subprocess.check_call(["openssl", "genrsa", "-out", cr.ca_key_file]) - - subprocess.check_call( - [ - "openssl", - "req", - "-new", - "-x509", - "-days", - "3650", - "-config", - cr.ca_conf, - "-key", - cr.ca_key_file, - "-out", - cr.ca_cert_file, - ] - ) - - ### CLIENT CERTIFICATE - - # Copy CA - shutil.copy(cr.ca_cert_file, f"{cr.client_certs_dir}/root.crt") - - for client in ["yugabytedb"]: - - crt_name = f"{cr.client_certs_dir}/{client}.crt" - key_name = f"{cr.client_certs_dir}/{client}.key" - - conf_name = f"{cr.ca_key_dir}/conf.client.{client}" - csr_name = f"{cr.ca_key_dir}/csr.client.{client}" - - if os.path.exists(crt_name) and os.path.exists(key_name): # No need to regenerate it - continue - - with open(conf_name, "w") as f: - f.write( - f"""[ req ] -prompt=no -distinguished_name = my_distinguished_name - -[ my_distinguished_name ] -organizationName = Yugabyte -commonName = {client} -""" - ) - - subprocess.check_call(["openssl", "genrsa", "-out", key_name]) - - subprocess.check_call( - [ - "openssl", - "req", - "-new", - "-config", - conf_name, - "-key", - key_name, - "-out", - csr_name, - ] - ) - - subprocess.check_call( - [ - "openssl", - "ca", - "-config", - cr.ca_conf, - "-keyfile", - cr.ca_key_file, - "-cert", - cr.ca_cert_file, - "-policy", - "my_policy", - "-out", - crt_name, - "-outdir", - cr.client_certs_dir, - "-in", - csr_name, - "-days", - "3650", - "-batch", - "-extfile", - conf_name, - ] - ) - - ### SERVERS - - for server_type in ["master", "tserver"]: - - folder = getattr(cr, f"{server_type}_certs_dir") - - # Copy CA - shutil.copy(cr.ca_cert_file, folder) - - for server_id in range(0, int(args.node_count)): - short_name = f"yb-{server_type}-{server_id}" - short_name_group = f"{short_name}.yb-{server_type}s" - full_name_group = f"{short_name}.{cr.namespace}.svc.cluster.local" - full_name = f"{short_name_group}.{cr.namespace}.svc.cluster.local" - - crt_name = f"{folder}/node.{full_name}.crt" - key_name = f"{folder}/node.{full_name}.key" - - conf_name = f"{cr.ca_key_dir}/conf.{full_name}" - csr_name = f"{cr.ca_key_dir}/csr.{full_name}" - - if os.path.exists(crt_name) and os.path.exists(key_name): # No need to regenerate it - continue - - with open(conf_name, "w") as f: - f.write( - f"""[ req ] -prompt=no -distinguished_name = my_distinguished_name - -[ my_distinguished_name ] -organizationName = Yugabyte -commonName = {full_name} - -# Multiple subject alternative names (SANs) such as IP Address, -# DNS Name, Email, URI, and so on, can be specified under this section -[ req_ext] -subjectAltName = @alt_names -[alt_names] -DNS.1 = {short_name} -DNS.2 = {full_name} -DNS.3 = {short_name_group} -DNS.4 = {full_name_group} -DNS.5 = yb-{server_type}s -DNS.6 = yb-{server_type}s.{cr.namespace} -DNS.7 = yb-{server_type}s.{cr.namespace}.svc.cluster.local -""" - ) - - subprocess.check_call(["openssl", "genrsa", "-out", key_name]) - - subprocess.check_call( - [ - "openssl", - "req", - "-new", - "-config", - conf_name, - "-key", - key_name, - "-out", - csr_name, - ] - ) - - subprocess.check_call( - [ - "openssl", - "ca", - "-config", - cr.ca_conf, - "-keyfile", - cr.ca_key_file, - "-cert", - cr.ca_cert_file, - "-policy", - "my_policy", - "-out", - crt_name, - "-outdir", - folder, - "-in", - csr_name, - "-days", - "3650", - "-batch", - "-extfile", - conf_name, - "-extensions", - "req_ext", - ] - ) - - -if __name__ == "__main__": - main() diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/README.md b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/README.md index 7c9370807..f681f9f6b 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/README.md +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/README.md @@ -128,4 +128,3 @@ Delete the resources: `kubectl delete -f test-app.yml`. 1. Delete all created resources from the cluster (eg. clean up test as described in the previous section.) 2. Make sure all load balancers and target groups have been removed. 3. Run `terraform destroy`. - diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf index 1ccd8bcda..1f9bdb651 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf @@ -79,7 +79,7 @@ resource "aws_eip" "ip_crdb" { tags = { Name = format("%s-ip-crdb%v", var.cluster_name, count.index) # Preserve mapping between ips and hostnames - ExpectedDNS = format("%s.%s", count.index, var.crdb_hostname_suffix) + ExpectedDNS = format("%s.%s", count.index, var.db_hostname_suffix) } } @@ -91,7 +91,7 @@ resource "aws_eip" "ip_yugabyte_masters" { tags = { Name = format("%s-ip-yugabyte-master%v", var.cluster_name, count.index) # Preserve mapping between ips and hostnames - ExpectedDNS = format("%s.master.%s", count.index, var.crdb_hostname_suffix) + ExpectedDNS = format("%s.master.%s", count.index, var.db_hostname_suffix) } } @@ -103,6 +103,6 @@ resource "aws_eip" "ip_yugabyte_tservers" { tags = { Name = format("%s-ip-yugabyte-tserver%v", var.cluster_name, count.index) # Preserve mapping between ips and hostnames - ExpectedDNS = format("%s.tserver.%s", count.index, var.crdb_hostname_suffix) + ExpectedDNS = format("%s.tserver.%s", count.index, var.db_hostname_suffix) } } diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf index 12cf2ba1a..192149f83 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf @@ -54,17 +54,18 @@ variable "app_hostname" { EOT } -variable "crdb_hostname_suffix" { +variable "db_hostname_suffix" { type = string description = <<-EOT - The domain name suffix shared by all of your CockroachDB nodes. - For instance, if your CRDB nodes were addressable at 0.db.example.com, - 1.db.example.com and 2.db.example.com, then the value would be db.example.com. + The domain name suffix shared by all of your databases nodes. + For instance, if your database nodes were addressable at 0.db.example.com, + 1.db.example.com and 2.db.example.com (CockroachDB) or 0.master.db.example.com, 1.tserver.db.example.com (Yugabyte), then the value would be db.example.com. Example: db.example.com EOT } + variable "datastore_type" { type = string description = <<-EOT diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf index d685caf58..74fb834ba 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf @@ -24,7 +24,7 @@ resource "local_file" "helm_chart_values" { replicas = length(var.crdb_internal_nodes) args = [ "--locality-advertise-addr=zone=${var.crdb_locality}@$(hostname -f)", - "--advertise-addr=$${HOSTNAME##*-}.${var.crdb_hostname_suffix}" + "--advertise-addr=$${HOSTNAME##*-}.${var.db_hostname_suffix}" ] } @@ -127,9 +127,9 @@ resource "local_file" "helm_chart_values" { } } }] - serverBroadcastAddress: "$${HOSTNAMENO}.master.${var.crdb_hostname_suffix}" - rpcBindAddress: "$${HOSTNAMENO}.master.${var.crdb_hostname_suffix}" - preCommands: "sed -E \"/\\.svc\\.cluster\\.local/ s/^([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)([[:space:]]+)/\\1 $(echo \"$${HOSTNAMENO}.master.${var.crdb_hostname_suffix}\" | sed 's/[\\/&]/\\\\&/g')\\2/\" /etc/hosts > /tmp/newhosts && /bin/cp /tmp/newhosts /etc/hosts && \\" + serverBroadcastAddress: "$${HOSTNAMENO}.master.${var.db_hostname_suffix}" + rpcBindAddress: "$${HOSTNAMENO}.master.${var.db_hostname_suffix}" + preCommands: "sed -E \"/\\.svc\\.cluster\\.local/ s/^([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)([[:space:]]+)/\\1 $(echo \"$${HOSTNAMENO}.master.${var.db_hostname_suffix}\" | sed 's/[\\/&]/\\\\&/g')\\2/\" /etc/hosts > /tmp/newhosts && /bin/cp /tmp/newhosts /etc/hosts && \\" } tserver = { @@ -141,9 +141,9 @@ resource "local_file" "helm_chart_values" { } } }] - serverBroadcastAddress: "$${HOSTNAMENO}.tserver.${var.crdb_hostname_suffix}" - rpcBindAddress: "$${HOSTNAMENO}.tserver.${var.crdb_hostname_suffix}" - preCommands: "sed -E \"/\\.svc\\.cluster\\.local/ s/^([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)([[:space:]]+)/\\1 $(echo \"$${HOSTNAMENO}.tserver.${var.crdb_hostname_suffix}\" | sed 's/[\\/&]/\\\\&/g')\\2/\" /etc/hosts > /tmp/newhosts && /bin/cp /tmp/newhosts /etc/hosts && \\" + serverBroadcastAddress: "$${HOSTNAMENO}.tserver.${var.db_hostname_suffix}" + rpcBindAddress: "$${HOSTNAMENO}.tserver.${var.db_hostname_suffix}" + preCommands: "sed -E \"/\\.svc\\.cluster\\.local/ s/^([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)([[:space:]]+)/\\1 $(echo \"$${HOSTNAMENO}.tserver.${var.db_hostname_suffix}\" | sed 's/[\\/&]/\\\\&/g')\\2/\" /etc/hosts > /tmp/newhosts && /bin/cp /tmp/newhosts /etc/hosts && \\" } gflags = { @@ -162,7 +162,7 @@ resource "local_file" "helm_chart_values" { } isMultiAz = true - masterAddresses = join(",", ["0.master.${var.crdb_hostname_suffix},1.master.${var.crdb_hostname_suffix},2.master.${var.crdb_hostname_suffix}", join(",", var.yugabyte_external_nodes)]) + masterAddresses = join(",", ["0.master.${var.db_hostname_suffix},1.master.${var.db_hostname_suffix},2.master.${var.db_hostname_suffix}", join(",", var.yugabyte_external_nodes)]) } loadBalancers = { diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf index 7f2981d40..240b35b29 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf @@ -24,7 +24,7 @@ resource "local_file" "dss_certs" { content = templatefile("${path.module}/templates/dss-certs.sh.tmp", { cluster_context = var.kubernetes_context_name namespace = var.kubernetes_namespace - crdb_hostname_suffix = var.crdb_hostname_suffix + db_hostname_suffix = var.db_hostname_suffix }) filename = "${local.workspace_location}/dss-certs.sh" } diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf index b02801730..f065f7489 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf @@ -4,7 +4,7 @@ resource "local_file" "tanka_config_main" { VAR_NAMESPACE = var.kubernetes_namespace VAR_CLUSTER_CONTEXT = var.kubernetes_context_name VAR_ENABLE_SCD = var.enable_scd - VAR_CRDB_HOSTNAME_SUFFIX = var.crdb_hostname_suffix + VAR_DB_HOSTNAME_SUFFIX = var.db_hostname_suffix VAR_CRDB_LOCALITY = var.crdb_locality VAR_CRDB_NODE_IPS = join(",", [for i in var.crdb_internal_nodes[*].ip : "'${i}'"]) VAR_INGRESS_NAME = var.ip_gateway diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp index 8ee81325b..9f3c26e42 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp @@ -14,4 +14,4 @@ else fi cd "$BASEDIR/../../../deploy/operations/certificates-management/" || exit 1 -./dss-certs.py --name ${cluster_context} --organization default_orga --cluster-context ${cluster_context} --nodes-public-address "..${crdb_hostname_suffix}" --namespace ${namespace} "$@" +./dss-certs.py --name ${cluster_context} --organization default_orga --cluster-context ${cluster_context} --nodes-public-address "..${db_hostname_suffix}" --namespace ${namespace} "$@" diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp index e9f6de78b..9bd9b3ca8 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp @@ -13,7 +13,7 @@ local metadata = metadataBase { enableScd: ${VAR_ENABLE_SCD}, // <-- This boolean value is VAR_ENABLE_SCD cockroach+: { image: '${VAR_CRDB_DOCKER_IMAGE_NAME}', - hostnameSuffix: '${VAR_CRDB_HOSTNAME_SUFFIX}', + hostnameSuffix: '${VAR_DB_HOSTNAME_SUFFIX}', locality: '${VAR_CRDB_LOCALITY}', nodeIPs: [${VAR_CRDB_NODE_IPS}], shouldInit: ${VAR_SHOULD_INIT}, diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf index 2cfdc68bf..83ba69568 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf @@ -11,17 +11,18 @@ variable "app_hostname" { EOT } -variable "crdb_hostname_suffix" { +variable "db_hostname_suffix" { type = string description = <<-EOT - The domain name suffix shared by all of your CockroachDB nodes. - For instance, if your CRDB nodes were addressable at 0.db.example.com, - 1.db.example.com and 2.db.example.com, then the value would be db.example.com. + The domain name suffix shared by all of your databases nodes. + For instance, if your database nodes were addressable at 0.db.example.com, + 1.db.example.com and 2.db.example.com (CockroachDB) or 0.master.db.example.com, 1.tserver.db.example.com (Yugabyte), then the value would be db.example.com. Example: db.example.com EOT } + variable "datastore_type" { type = string description = <<-EOT @@ -149,10 +150,13 @@ variable "should_init" { for a pool. When set true, this can initialize the data directories on your cluster, and prevent you from joining an existing pool. + Only used for CockroachDB with Tanka + Example: `true` EOT } + variable "desired_rid_db_version" { type = string description = <<-EOT diff --git a/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf b/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf index 4a6c925fa..cadca0f96 100644 --- a/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf +++ b/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf @@ -57,7 +57,7 @@ resource "google_compute_address" "ip_crdb" { region = local.region # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames. - description = format("%s.%s", count.index, var.crdb_hostname_suffix) + description = format("%s.%s", count.index, var.db_hostname_suffix) } # Static IP addresses for yugabyte instances @@ -67,7 +67,7 @@ resource "google_compute_address" "ip_yugabyte_masters" { region = local.region # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames. - description = format("%s.master.%s", count.index, var.crdb_hostname_suffix) + description = format("%s.master.%s", count.index, var.db_hostname_suffix) } resource "google_compute_address" "ip_yugabyte_tservers" { @@ -76,7 +76,7 @@ resource "google_compute_address" "ip_yugabyte_tservers" { region = local.region # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames. - description = format("%s.tserver.%s", count.index, var.crdb_hostname_suffix) + description = format("%s.tserver.%s", count.index, var.db_hostname_suffix) } locals { diff --git a/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf index 584adf2bb..aa5654c8b 100644 --- a/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf @@ -43,17 +43,18 @@ variable "app_hostname" { EOT } -variable "crdb_hostname_suffix" { +variable "db_hostname_suffix" { type = string description = <<-EOT - The domain name suffix shared by all of your CockroachDB nodes. - For instance, if your CRDB nodes were addressable at 0.db.example.com, - 1.db.example.com and 2.db.example.com, then the value would be db.example.com. + The domain name suffix shared by all of your databases nodes. + For instance, if your database nodes were addressable at 0.db.example.com, + 1.db.example.com and 2.db.example.com (CockroachDB) or 0.master.db.example.com, 1.tserver.db.example.com (Yugabyte), then the value would be db.example.com. Example: db.example.com EOT } + variable "datastore_type" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-aws-dss/README.md b/deploy/infrastructure/modules/terraform-aws-dss/README.md index 9d91288bd..191b3f712 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/README.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/README.md @@ -1,6 +1,6 @@ # terraform-aws-dss -This terraform module creates a Kubernetes cluster in Amazon Web Services using the Elastic Kubernetes Service (EKS) +This terraform module creates a Kubernetes cluster in Amazon Web Services using the Elastic Kubernetes Service (EKS) and generates the tanka files to deploy a DSS instance. @@ -15,7 +15,7 @@ Download & install the following tools to your workstation: 3. Install provider specific tools: 1. [Amazon Web Services](./README.md#amazon-web-services) - + #### Amazon Web Services 1. Install and initialize [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#getting-started-install-instructions). @@ -25,7 +25,7 @@ Download & install the following tools to your workstation: 1. We recommend to create an AWS_PROFILE using for instance `aws configure --profile aws-interuss-dss` Before running `terraform` commands, run once in your shell: `export AWS_PROFILE=aws-interuss-dss` Other methods are described here: https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration - + ### Deployment of the Kubernetes cluster @@ -54,8 +54,13 @@ It contains scripts to operate the cluster and setup the services. 1. Go to the new workspace `/build/workspace/${cluster_context}`. 2. Run `./get-credentials.sh` to login to kubernetes. You can now access the cluster with `kubectl`. -3. Generate the certificates using `./make-certs.sh`. Follow script instructions if you are not initializing the cluster. -4. Deploy the certificates using `./apply-certs.sh`. +3. If using CockroachDB: + 1. Generate the certificates using `./make-certs.sh`. Follow script instructions if you are not initializing the cluster. + 1. Deploy the certificates using `./apply-certs.sh`. +4. If using Yugabyte: + 1. Generate the certificates using `./dss-certs.sh init` + 1. If joining a cluster, check `dss-certs.sh`'s [help](../../../operations/certificates-management/README.md) to add others CA in your pool and share your CA with others pools members. + 1. Deploy the certificates using `./dss-certs.sh apply`. 5. Run `tk apply .` to deploy the services to kubernetes. (This may take up to 30 min) 6. Wait for services to initialize: - On AWS, load balancers and certificates are created by Kubernetes Operators. Therefore, it may take few minutes (~5min) to get the services up and running and generate the certificate. To track this progress, go to the following pages and check that: @@ -68,6 +73,6 @@ It contains scripts to operate the cluster and setup the services. 1. Note that the following operations can't be reverted and all data will be lost. 2. To delete all resources, run `tk delete .` in the workspace folder. -3. Make sure that all [load balancers](https://eu-west-1.console.aws.amazon.com/ec2/home#LoadBalancers:) and [target groups](https://eu-west-1.console.aws.amazon.com/ec2/home#TargetGroups:) have been deleted from the AWS region before next step. +3. Make sure that all [load balancers](https://eu-west-1.console.aws.amazon.com/ec2/home#LoadBalancers:) and [target groups](https://eu-west-1.console.aws.amazon.com/ec2/home#TargetGroups:) have been deleted from the AWS region before next step. 4. `terraform destroy` in your infrastructure folder. -5. On the [EBS page](https://eu-west-1.console.aws.amazon.com/ec2/home#Volumes:), make sure to manually clean up the persistent storage. Note that the correct AWS region shall be selected. +5. On the [EBS page](https://eu-west-1.console.aws.amazon.com/ec2/home#Volumes:), make sure to manually clean up the persistent storage. Note that the correct AWS region shall be selected. diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md index d8b6fccf5..1ba18ebd7 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md @@ -56,12 +56,12 @@ Leave empty to disable record creation.

Example: dss.example.com

- crdb_hostname_suffix + db_hostname_suffix string -

The domain name suffix shared by all of your CockroachDB nodes. -For instance, if your CRDB nodes were addressable at 0.db.example.com, -1.db.example.com and 2.db.example.com, then the value would be db.example.com.

+

The domain name suffix shared by all of your databases nodes. +For instance, if your database nodes were addressable at 0.db.example.com, +1.db.example.com and 2.db.example.com (CockroachDB) or 0.master.db.example.com, 1.tserver.db.example.com (Yugabyte), then the value would be db.example.com.

Example: db.example.com

@@ -189,6 +189,7 @@ Example:

Set to false if joining an existing pool, true if creating the first DSS instance for a pool. When set true, this can initialize the data directories on your cluster, and prevent you from joining an existing pool.

+

Only used for CockroachDB with Tanka

Example: true

diff --git a/deploy/infrastructure/modules/terraform-aws-dss/main.tf b/deploy/infrastructure/modules/terraform-aws-dss/main.tf index 9000f9ca4..4f7257d0c 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/main.tf @@ -3,7 +3,7 @@ module "terraform-aws-kubernetes" { cluster_name = var.cluster_name aws_region = var.aws_region app_hostname = var.app_hostname - crdb_hostname_suffix = var.crdb_hostname_suffix + db_hostname_suffix = var.db_hostname_suffix datastore_type = var.datastore_type aws_instance_type = var.aws_instance_type aws_route53_zone_id = var.aws_route53_zone_id @@ -23,7 +23,7 @@ module "terraform-commons-dss" { app_hostname = var.app_hostname crdb_image_tag = var.crdb_image_tag crdb_cluster_name = var.crdb_cluster_name - crdb_hostname_suffix = var.crdb_hostname_suffix + db_hostname_suffix = var.db_hostname_suffix datastore_type = var.datastore_type should_init = var.should_init authorization = var.authorization diff --git a/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars b/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars index 718ab40bd..7b159f15d 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars +++ b/deploy/infrastructure/modules/terraform-aws-dss/terraform.dev.example.tfvars @@ -9,7 +9,7 @@ aws_route53_zone_id = "Z01551234567890123456" # Hostnames app_hostname = "dss.interuss.example.com" -crdb_hostname_suffix = "db.interuss.example.com" +db_hostname_suffix = "db.interuss.example.com" # Kubernetes configuration cluster_name = "dss-dev-ew1" @@ -23,10 +23,19 @@ image = "docker.io/interuss/dss:latest" authorization = { public_key_pem_path = "/test-certs/auth2.pem" } -should_init = true + +# Datastore +datastore_type = "cockroachdb" # CockroachDB crdb_image_tag = "v24.1.3" crdb_cluster_name = "interuss_example" crdb_locality = "interuss_dss-aws-ew1" crdb_external_nodes = [] +should_init = true + +# Yugabyte +yugabyte_region = "aws-uss-1" +yugabyte_zone = "aws-uss-1" +yugabyte_light_resources = false +yugabyte_external_nodes = [] diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf index 7cb4337d7..fd1b51a7f 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf @@ -54,17 +54,18 @@ variable "app_hostname" { EOT } -variable "crdb_hostname_suffix" { +variable "db_hostname_suffix" { type = string description = <<-EOT - The domain name suffix shared by all of your CockroachDB nodes. - For instance, if your CRDB nodes were addressable at 0.db.example.com, - 1.db.example.com and 2.db.example.com, then the value would be db.example.com. + The domain name suffix shared by all of your databases nodes. + For instance, if your database nodes were addressable at 0.db.example.com, + 1.db.example.com and 2.db.example.com (CockroachDB) or 0.master.db.example.com, 1.tserver.db.example.com (Yugabyte), then the value would be db.example.com. Example: db.example.com EOT } + variable "datastore_type" { type = string description = <<-EOT @@ -243,10 +244,13 @@ variable "should_init" { for a pool. When set true, this can initialize the data directories on your cluster, and prevent you from joining an existing pool. + Only used for CockroachDB with Tanka + Example: `true` EOT } + variable "desired_rid_db_version" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-google-dss/README.md b/deploy/infrastructure/modules/terraform-google-dss/README.md index 62b6c6dbc..5070da4fb 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/README.md +++ b/deploy/infrastructure/modules/terraform-google-dss/README.md @@ -1,6 +1,6 @@ # terraform-google-dss -This terraform module creates a Kubernetes cluster in Google Cloud Engine and generates +This terraform module creates a Kubernetes cluster in Google Cloud Engine and generates the tanka files to deploy a DSS instance. ## Getting started @@ -12,7 +12,7 @@ Download & install the following tools to your workstation: 2. Install tools from [Prerequisites](../../../../build/README.md) 3. Install provider specific tools: 1. [Google Cloud Engine](./README.md#google-cloud-engine) - + #### Google Cloud Engine 1. Install and initialize [Google Cloud CLI](https://cloud.google.com/sdk/docs/install-sdk). @@ -41,7 +41,7 @@ Download & install the following tools to your workstation: 6. In the new directory (ie /deploy/infrastructure/personal/terraform-google-dss-dev), initialize terraform: `terraform init`. 7. Run `terraform plan` to check that the configuration is valid. It will display the resources which will be provisioned. 8. Run `terraform apply` to deploy the cluster. (This operation may take up to 15 min.) -9. Configure the DNS resolution to the public ip addresses. DNS entries can be either managed manually or +9. Configure the DNS resolution to the public ip addresses. DNS entries can be either managed manually or handled by terraform depending on the cloud provider. See [DNS](DNS.md) for details. ## Deployment of the DSS services @@ -54,8 +54,13 @@ It contains scripts to operate the cluster and setup the services. 1. Go to the new workspace `/build/workspace/${cluster_context}`. 2. Run `./get-credentials.sh` to login to kubernetes. You can now access the cluster with `kubectl`. -3. Generate the certificates using `./make-certs.sh`. Follow script instructions if you are not initializing the cluster. -4. Deploy the certificates using `./apply-certs.sh`. +3. If using CockroachDB: + 1. Generate the certificates using `./make-certs.sh`. Follow script instructions if you are not initializing the cluster. + 1. Deploy the certificates using `./apply-certs.sh`. +4. If using Yugabyte: + 1. Generate the certificates using `./dss-certs.sh init` + 1. If joining a cluster, check `dss-certs.sh`'s [help](../../../operations/certificates-management/README.md) to add others CA in your pool and share your CA with others pools members. + 1. Deploy the certificates using `./dss-certs.sh apply`. 5. Run `tk apply .` to deploy the services to kubernetes. (This may take up to 30 min) 6. Wait for services to initialize: - On Google Cloud, the highest-latency operation is provisioning of the HTTPS certificate which generally takes 10-45 minutes. To track this progress: @@ -71,4 +76,4 @@ It contains scripts to operate the cluster and setup the services. To delete all resources, run `terraform destroy`. Note that this operation can't be reverted and all data will be lost. -For Google Cloud Engine, make sure to manually clean up the persistent storage: https://console.cloud.google.com/compute/disks +For Google Cloud Engine, make sure to manually clean up the persistent storage: https://console.cloud.google.com/compute/disks diff --git a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md index eeb649317..56cb99bd8 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md @@ -52,12 +52,12 @@ Example: n2-standard-4 for production, e2-medium for d

Example: dss.example.com

- crdb_hostname_suffix + db_hostname_suffix string -

The domain name suffix shared by all of your CockroachDB nodes. -For instance, if your CRDB nodes were addressable at 0.db.example.com, -1.db.example.com and 2.db.example.com, then the value would be db.example.com.

+

The domain name suffix shared by all of your databases nodes. +For instance, if your database nodes were addressable at 0.db.example.com, +1.db.example.com and 2.db.example.com (CockroachDB) or 0.master.db.example.com, 1.tserver.db.example.com (Yugabyte), then the value would be db.example.com.

Example: db.example.com

@@ -185,6 +185,7 @@ Example:

Set to false if joining an existing pool, true if creating the first DSS instance for a pool. When set true, this can initialize the data directories on your cluster, and prevent you from joining an existing pool.

+

Only used for CockroachDB with Tanka

Example: true

diff --git a/deploy/infrastructure/modules/terraform-google-dss/main.tf b/deploy/infrastructure/modules/terraform-google-dss/main.tf index 8b2782dd4..33d370f65 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/main.tf @@ -4,7 +4,7 @@ module "terraform-google-kubernetes" { cluster_name = var.cluster_name google_zone = var.google_zone app_hostname = var.app_hostname - crdb_hostname_suffix = var.crdb_hostname_suffix + db_hostname_suffix = var.db_hostname_suffix datastore_type = var.datastore_type google_dns_managed_zone_name = var.google_dns_managed_zone_name google_machine_type = var.google_machine_type @@ -22,7 +22,7 @@ module "terraform-commons-dss" { app_hostname = var.app_hostname crdb_image_tag = var.crdb_image_tag crdb_cluster_name = var.crdb_cluster_name - crdb_hostname_suffix = var.crdb_hostname_suffix + db_hostname_suffix = var.db_hostname_suffix datastore_type = var.datastore_type should_init = var.should_init authorization = var.authorization diff --git a/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars b/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars index 3a22909ca..3aa7ca011 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars +++ b/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars @@ -9,7 +9,7 @@ google_zone = "europe-west6-a" # DNS google_dns_managed_zone_name = "interuss-example-com" app_hostname = "dss.interuss.example.com" -crdb_hostname_suffix = "db.interuss.example.com" +db_hostname_suffix = "db.interuss.example.com" # Kubernetes configuration cluster_name = "dss-dev-w6a" @@ -24,7 +24,6 @@ image_pull_secret = "" authorization = { public_key_pem_path = "/test-certs/auth2.pem" } -should_init = true # Datastore datastore_type = "cockroachdb" @@ -34,6 +33,10 @@ crdb_image_tag = "v24.1.3" crdb_cluster_name = "interuss_example" crdb_locality = "interuss_dss-dev-w6a" crdb_external_nodes = [] +should_init = true # Yugabyte -yugabyte_region = "uss-1" +yugabyte_region = "gcp-uss-1" +yugabyte_zone = "gcp-uss-1" +yugabyte_light_resources = false +yugabyte_external_nodes = [] diff --git a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf index 86d54c973..7dbbff280 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf @@ -43,17 +43,18 @@ variable "app_hostname" { EOT } -variable "crdb_hostname_suffix" { +variable "db_hostname_suffix" { type = string description = <<-EOT - The domain name suffix shared by all of your CockroachDB nodes. - For instance, if your CRDB nodes were addressable at 0.db.example.com, - 1.db.example.com and 2.db.example.com, then the value would be db.example.com. + The domain name suffix shared by all of your databases nodes. + For instance, if your database nodes were addressable at 0.db.example.com, + 1.db.example.com and 2.db.example.com (CockroachDB) or 0.master.db.example.com, 1.tserver.db.example.com (Yugabyte), then the value would be db.example.com. Example: db.example.com EOT } + variable "datastore_type" { type = string description = <<-EOT @@ -232,10 +233,13 @@ variable "should_init" { for a pool. When set true, this can initialize the data directories on your cluster, and prevent you from joining an existing pool. + Only used for CockroachDB with Tanka + Example: `true` EOT } + variable "desired_rid_db_version" { type = string description = <<-EOT diff --git a/deploy/infrastructure/utils/definitions/crdb_hostname_suffix.tf b/deploy/infrastructure/utils/definitions/crdb_hostname_suffix.tf deleted file mode 100644 index 005d0d86f..000000000 --- a/deploy/infrastructure/utils/definitions/crdb_hostname_suffix.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "crdb_hostname_suffix" { - type = string - description = <<-EOT - The domain name suffix shared by all of your CockroachDB nodes. - For instance, if your CRDB nodes were addressable at 0.db.example.com, - 1.db.example.com and 2.db.example.com, then the value would be db.example.com. - - Example: db.example.com - EOT -} \ No newline at end of file diff --git a/deploy/infrastructure/utils/definitions/db_hostname_suffix.tf b/deploy/infrastructure/utils/definitions/db_hostname_suffix.tf new file mode 100644 index 000000000..7770b8609 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/db_hostname_suffix.tf @@ -0,0 +1,10 @@ +variable "db_hostname_suffix" { + type = string + description = <<-EOT + The domain name suffix shared by all of your databases nodes. + For instance, if your database nodes were addressable at 0.db.example.com, + 1.db.example.com and 2.db.example.com (CockroachDB) or 0.master.db.example.com, 1.tserver.db.example.com (Yugabyte), then the value would be db.example.com. + + Example: db.example.com + EOT +} diff --git a/deploy/infrastructure/utils/definitions/should_init.tf b/deploy/infrastructure/utils/definitions/should_init.tf index bdbf936df..e61d74259 100644 --- a/deploy/infrastructure/utils/definitions/should_init.tf +++ b/deploy/infrastructure/utils/definitions/should_init.tf @@ -5,6 +5,8 @@ variable "should_init" { for a pool. When set true, this can initialize the data directories on your cluster, and prevent you from joining an existing pool. + Only used for CockroachDB with Tanka + Example: `true` EOT -} \ No newline at end of file +} diff --git a/deploy/infrastructure/utils/definitions/yugabyte_resources.tf b/deploy/infrastructure/utils/definitions/yugabyte_resources.tf deleted file mode 100644 index abd429c92..000000000 --- a/deploy/infrastructure/utils/definitions/yugabyte_resources.tf +++ /dev/null @@ -1,11 +0,0 @@ -variable "should_init" { - type = bool - description = <<-EOT - Set to false if joining an existing pool, true if creating the first DSS instance - for a pool. When set true, this can initialize the data directories on your cluster, - and prevent you from joining an existing pool. - - Example: `true` - EOT -} - diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 732c867c1..515137d10 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -23,7 +23,7 @@ # Variables per project # For all */terraform-* -GLOBAL_VARIABLES = ["app_hostname", "crdb_hostname_suffix", "datastore_type"] +GLOBAL_VARIABLES = ["app_hostname", "db_hostname_suffix", "datastore_type"] # dependencies/terraform-commons-dss COMMONS_DSS_VARIABLES = GLOBAL_VARIABLES + [ diff --git a/deploy/operations/ci/aws-1/main.tf b/deploy/operations/ci/aws-1/main.tf index 998b71eb9..fd238a02b 100644 --- a/deploy/operations/ci/aws-1/main.tf +++ b/deploy/operations/ci/aws-1/main.tf @@ -19,7 +19,7 @@ module "terraform-aws-dss" { cluster_name = var.cluster_name crdb_image_tag = var.crdb_image_tag crdb_cluster_name = var.crdb_cluster_name - crdb_hostname_suffix = var.crdb_hostname_suffix + db_hostname_suffix = var.db_hostname_suffix crdb_locality = var.crdb_locality crdb_external_nodes = var.crdb_external_nodes image = var.image diff --git a/deploy/operations/ci/aws-1/terraform.tfvars b/deploy/operations/ci/aws-1/terraform.tfvars index 22be80d06..b9e935286 100644 --- a/deploy/operations/ci/aws-1/terraform.tfvars +++ b/deploy/operations/ci/aws-1/terraform.tfvars @@ -8,7 +8,7 @@ aws_route53_zone_id = "Z03377073HUSGB4L9FKEK" # Hostnames app_hostname = "dss.ci.aws-interuss.uspace.dev" -crdb_hostname_suffix = "db.ci.aws-interuss.uspace.dev" +db_hostname_suffix = "db.ci.aws-interuss.uspace.dev" # Kubernetes configuration kubernetes_version = 1.32 diff --git a/deploy/operations/ci/aws-1/variables.gen.tf b/deploy/operations/ci/aws-1/variables.gen.tf index 7cb4337d7..fd1b51a7f 100644 --- a/deploy/operations/ci/aws-1/variables.gen.tf +++ b/deploy/operations/ci/aws-1/variables.gen.tf @@ -54,17 +54,18 @@ variable "app_hostname" { EOT } -variable "crdb_hostname_suffix" { +variable "db_hostname_suffix" { type = string description = <<-EOT - The domain name suffix shared by all of your CockroachDB nodes. - For instance, if your CRDB nodes were addressable at 0.db.example.com, - 1.db.example.com and 2.db.example.com, then the value would be db.example.com. + The domain name suffix shared by all of your databases nodes. + For instance, if your database nodes were addressable at 0.db.example.com, + 1.db.example.com and 2.db.example.com (CockroachDB) or 0.master.db.example.com, 1.tserver.db.example.com (Yugabyte), then the value would be db.example.com. Example: db.example.com EOT } + variable "datastore_type" { type = string description = <<-EOT @@ -243,10 +244,13 @@ variable "should_init" { for a pool. When set true, this can initialize the data directories on your cluster, and prevent you from joining an existing pool. + Only used for CockroachDB with Tanka + Example: `true` EOT } + variable "desired_rid_db_version" { type = string description = <<-EOT diff --git a/deploy/services/tanka/examples/minimum/main.jsonnet b/deploy/services/tanka/examples/minimum/main.jsonnet index 52005bc7b..208887ddf 100644 --- a/deploy/services/tanka/examples/minimum/main.jsonnet +++ b/deploy/services/tanka/examples/minimum/main.jsonnet @@ -13,7 +13,7 @@ local metadata = metadataBase { enableScd: false, // <-- This boolean value is VAR_ENABLE_SCD cockroach+: { image: 'VAR_CRDB_DOCKER_IMAGE_NAME', - hostnameSuffix: 'VAR_CRDB_HOSTNAME_SUFFIX', + hostnameSuffix: 'VAR_DB_HOSTNAME_SUFFIX', locality: 'VAR_CRDB_LOCALITY', nodeIPs: ['VAR_CRDB_NODE_IP1', 'VAR_CRDB_NODE_IP2', 'VAR_CRDB_NODE_IP3'], shouldInit: false, // <-- This boolean value is VAR_SHOULD_INIT