diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf index 3ec55966b..99a97c717 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf @@ -65,15 +65,23 @@ variable "crdb_hostname_suffix" { EOT } -variable "cluster_name" { +variable "datastore_type" { type = string description = <<-EOT - Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted) + Type of datastore used - Example: `dss-che-1` + Supported technologies: cockroachdb, yugabyte EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" } + variable "node_count" { type = number description = <<-EOT @@ -84,12 +92,21 @@ variable "node_count" { EOT validation { - condition = contains([1, 3], var.node_count) - error_message = "Currently, only 1 node or 3 nodes deployments are supported." + condition = (var.datastore_type == "cockroach" && contains([1, 3], var.node_count)) || (var.datastore_type == "yugabyte" && var.node_count > 0) + error_message = "Currently, only 1 node or 3 nodes deployments are supported for CockroachDB. If you use Yugabyte, you need to have at least one node." } } +variable "cluster_name" { + type = string + description = <<-EOT + Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted) + + Example: `dss-che-1` + EOT +} + variable "kubernetes_version" { type = string description = <<-EOT diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf index fc5b73ec0..d799f1f0c 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf @@ -2,11 +2,15 @@ locals { # Tanka defines itself the variable below. For helm, since we are using the official helm CRDB chart, # the following variable has to be provided here. helm_crdb_statefulset_name = "dss-cockroachdb" + + # This pre command is used bellow in yugabyte deployments to make the local ip pointing to the public hostname we want to use, until https://github.com/yugabyte/yugabyte-db/issues/27367 is fixed + yugabyte_precommand_prefix = "sed -E \"/\\.svc\\.cluster\\.local/ s/^([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)([[:space:]]+)/\\1 $(echo \"$${HOSTNAMENO}." + yugabyte_precommand_suffix = ".${var.crdb_hostname_suffix}\" | sed 's/[\\/&]/\\\\&/g')\\2/\" /etc/hosts > /tmp/newhosts && /bin/cp /tmp/newhosts /etc/hosts && \\" } resource "local_file" "helm_chart_values" { filename = "${local.workspace_location}/helm_values.yml" - content = yamlencode({ + content = var.datastore_type == "cockroachdb" ? yamlencode({ cockroachdb = { image = { tag = var.crdb_image_tag @@ -66,8 +70,134 @@ resource "local_file" "helm_chart_values" { } } + global = { + cloudProvider = var.kubernetes_cloud_provider_name + } + }) : yamlencode({ + cockroachdb = { + enabled = false + } + yugabyte = { + enabled = true + + resource = var.yugabyte_light_resources ? { + master = { + requests = { + cpu = "0.1" + memory = "0.5G" + } + } + tserver = { + requests = { + cpu = "0.1" + memory = "0.5G" + } + } + } : {} + enableLoadBalancer = false + + replicas = { + master = var.node_count + tserver = var.node_count + totalMasters = length(var.yugabyte_external_nodes) + var.node_count + } + + master = { + extraEnv = [{ + name = "HOSTNAMENO" + valueFrom = { + fieldRef = { + fieldPath = "metadata.labels['apps.kubernetes.io/pod-index']" + } + } + }] + serverBroadcastAddress : "$${HOSTNAMENO}.master.${var.crdb_hostname_suffix}" + rpcBindAddress : "$${HOSTNAMENO}.master.${var.crdb_hostname_suffix}" + advanced = { + preCommands : "${local.yugabyte_precommand_prefix}master${local.yugabyte_precommand_suffix}" + } + } + + tserver = { + extraEnv = [{ + name = "HOSTNAMENO" + valueFrom = { + fieldRef = { + fieldPath = "metadata.labels['apps.kubernetes.io/pod-index']" + } + } + }] + serverBroadcastAddress : "$${HOSTNAMENO}.tserver.${var.crdb_hostname_suffix}" + rpcBindAddress : "$${HOSTNAMENO}.tserver.${var.crdb_hostname_suffix}" + advanced = { + preCommands : "${local.yugabyte_precommand_prefix}tserver${local.yugabyte_precommand_suffix}" + } + } + + gflags = { + master = { + placement_cloud : var.yugabyte_cloud + placement_region : var.yugabyte_region + placement_zone : var.yugabyte_zone + use_private_ip : "zone" + } + tserver = { + placement_cloud : var.yugabyte_cloud + placement_region : var.yugabyte_region + placement_zone : var.yugabyte_zone + use_private_ip : "zone" + } + } + + masterAddresses = join(",", concat([ + for i in range(var.node_count) : format("%s.master.${var.crdb_hostname_suffix}", i) + ], var.yugabyte_external_nodes)) + } + + loadBalancers = { + cockroachdbNodes = [] + + yugabyteMasterNodes = [ + for ip in var.yugabyte_internal_masters_nodes[*].ip : + { + ip = ip + subnet = var.workload_subnet + } + ] + + yugabyteTserverNodes = [ + for ip in var.yugabyte_internal_tservers_nodes[*].ip : + { + ip = ip + subnet = var.workload_subnet + } + ] + + dssGateway = { + ip = var.ip_gateway + subnet = var.workload_subnet + certName = var.gateway_cert_name + sslPolicy = var.ssl_policy + } + } + + dss = { + image = var.image + + conf = { + pubKeys = [ + "/test-certs/auth2.pem" + ] + jwksEndpoint = var.authorization.jwks != null ? var.authorization.jwks.endpoint : "" + jwksKeyIds = var.authorization.jwks != null ? [var.authorization.jwks.key_id] : [] + hostname = var.app_hostname + enableScd = var.enable_scd + } + } + global = { cloudProvider = var.kubernetes_cloud_provider_name } }) + } diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf index 01ee71f17..e7f748d36 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf @@ -1,5 +1,6 @@ resource "local_file" "make_certs" { + count = var.datastore_type == "cockroachdb" ? 1 : 0 content = templatefile("${path.module}/templates/make-certs.sh.tmp", { cluster_context = var.kubernetes_context_name namespace = var.kubernetes_namespace @@ -10,6 +11,7 @@ resource "local_file" "make_certs" { } resource "local_file" "apply_certs" { + count = var.datastore_type == "cockroachdb" ? 1 : 0 content = templatefile("${path.module}/templates/apply-certs.sh.tmp", { cluster_context = var.kubernetes_context_name namespace = var.kubernetes_namespace @@ -17,6 +19,17 @@ resource "local_file" "apply_certs" { filename = "${local.workspace_location}/apply-certs.sh" } +resource "local_file" "dss_certs" { + count = var.datastore_type == "yugabyte" ? 1 : 0 + content = templatefile("${path.module}/templates/dss-certs.sh.tmp", { + cluster_context = var.kubernetes_context_name + namespace = var.kubernetes_namespace + crdb_hostname_suffix = var.crdb_hostname_suffix + node_count = var.node_count + }) + filename = "${local.workspace_location}/dss-certs.sh" +} + resource "local_file" "get_credentials" { content = templatefile("${path.module}/templates/get-credentials.sh.tmp", { get_credentials_cmd = var.kubernetes_get_credentials_cmd diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp new file mode 100644 index 000000000..be4d29a5c --- /dev/null +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# This file was automatically generated by terraform-commons-dss. +# Do not edit it directly. + +set -eo pipefail + +OS=$(uname) +if [[ "$OS" == "Darwin" ]]; then + # OSX uses BSD readlink + BASEDIR="$(dirname "$0")" +else + BASEDIR=$(readlink -e "$(dirname "$0")") +fi +cd "$BASEDIR/../../../deploy/operations/certificates-management/" || exit 1 + +./dss-certs.py --name ${cluster_context} --organization default_orga --cluster-context ${cluster_context} --nodes-public-address "..${crdb_hostname_suffix}" --namespace ${namespace} --nodes-count ${node_count} "$@" diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf index 20bbc304a..b6f47113a 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf @@ -22,6 +22,39 @@ variable "crdb_hostname_suffix" { EOT } +variable "datastore_type" { + type = string + description = <<-EOT + Type of datastore used + + Supported technologies: cockroachdb, yugabyte + EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" +} + + +variable "node_count" { + type = number + description = <<-EOT + Number of Kubernetes nodes which should correspond to the desired CockroachDB nodes. + Currently, only single node or three nodes deployments are supported. + + Example: `3` + EOT + + validation { + condition = (var.datastore_type == "cockroach" && contains([1, 3], var.node_count)) || (var.datastore_type == "yugabyte" && var.node_count > 0) + error_message = "Currently, only 1 node or 3 nodes deployments are supported for CockroachDB. If you use Yugabyte, you need to have at least one node." + } +} + + variable "image" { type = string description = <<-EOT @@ -225,3 +258,62 @@ variable "kubernetes_namespace" { } } +variable "yugabyte_cloud" { + type = string + description = <<-EOT + Cloud of yugabyte instances, used for partionning. + + Should be set to dss unless you're doing advanced partitionning. + EOT + + default = "dss" +} + + +variable "yugabyte_region" { + type = string + description = <<-EOT + Region of yugabyte instances, used for partionning. + + Should be different from others USS in a cluster. + EOT + + default = "uss-1" +} + + +variable "yugabyte_zone" { + type = string + description = <<-EOT + Zone of yugabyte instances, used for partionning. + + Should be set to zone unless you're doing advanced partitionning. + EOT + + default = "zone" +} + + +variable "yugabyte_light_resources" { + type = bool + description = <<-EOT + Enable light resources reservation for yugabyte instances. + + Useful for a dev cluster when you don't want to overload your kubernetes cluster. + EOT + + default = false +} + + +variable "yugabyte_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. + Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"] + EOT + default = [] +} + + + diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables_internal.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables_internal.tf index 0c6b28b3e..d56ca25f1 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables_internal.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables_internal.tf @@ -29,6 +29,22 @@ variable "crdb_internal_nodes" { description = "List of the IP addresses and related dns for the Cockroach DB nodes" } +variable "yugabyte_internal_masters_nodes" { + type = list(object({ + dns = string + ip = string + })) + description = "List of the IP addresses and related dns for the Yugabyte DB master nodes" +} + +variable "yugabyte_internal_tservers_nodes" { + type = list(object({ + dns = string + ip = string + })) + description = "List of the IP addresses and related dns for the Yugabyte DB tserver nodes" +} + variable "ip_gateway" { type = string description = "IP of the gateway used by the DSS service" diff --git a/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf b/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf index cb0416c01..4a6c925fa 100644 --- a/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf +++ b/deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf @@ -52,7 +52,7 @@ resource "google_compute_global_address" "ip_gateway" { # Static IP addresses for CRDB instances resource "google_compute_address" "ip_crdb" { - count = var.node_count + count = var.datastore_type == "cockroachdb" ? var.node_count : 0 name = format("%s-ip-crdb%v", var.cluster_name, count.index) region = local.region @@ -60,6 +60,25 @@ resource "google_compute_address" "ip_crdb" { description = format("%s.%s", count.index, var.crdb_hostname_suffix) } +# Static IP addresses for yugabyte instances +resource "google_compute_address" "ip_yugabyte_masters" { + count = var.datastore_type == "yugabyte" ? var.node_count : 0 + name = format("%s-ip-yugabyte-master%v", var.cluster_name, count.index) + region = local.region + + # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames. + description = format("%s.master.%s", count.index, var.crdb_hostname_suffix) +} + +resource "google_compute_address" "ip_yugabyte_tservers" { + count = var.datastore_type == "yugabyte" ? var.node_count : 0 + name = format("%s-ip-yugabyte-tserver%v", var.cluster_name, count.index) + region = local.region + + # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames. + description = format("%s.tserver.%s", count.index, var.crdb_hostname_suffix) +} + locals { kubectl_cluster_context_name = format("gke_%s_%s_%s", google_container_cluster.kubernetes_cluster.project, google_container_cluster.kubernetes_cluster.location, google_container_cluster.kubernetes_cluster.name) } diff --git a/deploy/infrastructure/dependencies/terraform-google-kubernetes/dns.tf b/deploy/infrastructure/dependencies/terraform-google-kubernetes/dns.tf index 85147bcc8..3a9b7f4c1 100644 --- a/deploy/infrastructure/dependencies/terraform-google-kubernetes/dns.tf +++ b/deploy/infrastructure/dependencies/terraform-google-kubernetes/dns.tf @@ -16,11 +16,31 @@ resource "google_dns_record_set" "gateway" { } resource "google_dns_record_set" "crdb" { - count = var.google_dns_managed_zone_name == "" ? 0 : var.node_count + count = var.datastore_type == "cockroachdb" && var.google_dns_managed_zone_name != "" ? var.node_count : 0 name = "${google_compute_address.ip_crdb[count.index].description}." # description contains the expected hostname type = "A" ttl = 300 managed_zone = data.google_dns_managed_zone.default[0].name rrdatas = [google_compute_address.ip_crdb[count.index].address] -} \ No newline at end of file +} + +resource "google_dns_record_set" "yugabyte_masters" { + count = var.datastore_type == "yugabyte" && var.google_dns_managed_zone_name != "" ? var.node_count : 0 + name = "${google_compute_address.ip_yugabyte_masters[count.index].description}." # description contains the expected hostname + type = "A" + ttl = 300 + + managed_zone = data.google_dns_managed_zone.default[0].name + rrdatas = [google_compute_address.ip_yugabyte_masters[count.index].address] +} + +resource "google_dns_record_set" "yugabyte_tserver" { + count = var.datastore_type == "yugabyte" && var.google_dns_managed_zone_name != "" ? var.node_count : 0 + name = "${google_compute_address.ip_yugabyte_tservers[count.index].description}." # description contains the expected hostname + type = "A" + ttl = 300 + + managed_zone = data.google_dns_managed_zone.default[0].name + rrdatas = [google_compute_address.ip_yugabyte_tservers[count.index].address] +} diff --git a/deploy/infrastructure/dependencies/terraform-google-kubernetes/output.tf b/deploy/infrastructure/dependencies/terraform-google-kubernetes/output.tf index aa2a27cda..b239e572b 100644 --- a/deploy/infrastructure/dependencies/terraform-google-kubernetes/output.tf +++ b/deploy/infrastructure/dependencies/terraform-google-kubernetes/output.tf @@ -2,6 +2,14 @@ output "crdb_addresses" { value = [for a in google_compute_address.ip_crdb[*] : { expected_dns : a.description, address : a.address }] } +output "yugabyte_masters_addresses" { + value = [for a in google_compute_address.ip_yugabyte_masters[*] : { expected_dns : a.description, address : a.address }] +} + +output "yugabyte_tservers_addresses" { + value = [for a in google_compute_address.ip_yugabyte_tservers[*] : { expected_dns : a.description, address : a.address }] +} + output "gateway_address" { value = { expected_dns : google_compute_global_address.ip_gateway.description, @@ -41,3 +49,21 @@ output "crdb_nodes" { } ] } + +output "yugabyte_masters_nodes" { + value = [ + for i in google_compute_address.ip_yugabyte_masters : { + ip = i.address + dns = i.description + } + ] +} + +output "yugabyte_tservers_nodes" { + value = [ + for i in google_compute_address.ip_yugabyte_tservers : { + ip = i.address + dns = i.description + } + ] +} diff --git a/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf index 07d364c55..753908911 100644 --- a/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-google-kubernetes/variables.gen.tf @@ -54,15 +54,23 @@ variable "crdb_hostname_suffix" { EOT } -variable "cluster_name" { +variable "datastore_type" { type = string description = <<-EOT - Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted) + Type of datastore used - Example: `dss-che-1` + Supported technologies: cockroachdb, yugabyte EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" } + variable "node_count" { type = number description = <<-EOT @@ -73,12 +81,21 @@ variable "node_count" { EOT validation { - condition = contains([1, 3], var.node_count) - error_message = "Currently, only 1 node or 3 nodes deployments are supported." + condition = (var.datastore_type == "cockroach" && contains([1, 3], var.node_count)) || (var.datastore_type == "yugabyte" && var.node_count > 0) + error_message = "Currently, only 1 node or 3 nodes deployments are supported for CockroachDB. If you use Yugabyte, you need to have at least one node." } } +variable "cluster_name" { + type = string + description = <<-EOT + Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted) + + Example: `dss-che-1` + EOT +} + variable "kubernetes_version" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md index 8e28f676f..ef1c1072f 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md @@ -65,11 +65,11 @@ For instance, if your CRDB nodes were addressable at 0.db.example.com,

Example: db.example.com

- cluster_name + datastore_type string - -

Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)

-

Example: dss-che-1

+ "cockroachdb" +

Type of datastore used

+

Supported technologies: cockroachdb, yugabyte

node_count @@ -78,6 +78,13 @@ For instance, if your CRDB nodes were addressable at 0.db.example.com,

Number of Kubernetes nodes which should correspond to the desired CockroachDB nodes. Currently, only single node or three nodes deployments are supported.

Example: 3

+ + + cluster_name + string + +

Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)

+

Example: dss-che-1

kubernetes_version @@ -243,6 +250,41 @@ Example: ["0.db.dss.example.com", "1.db.dss.example.com", &q "default"

Namespace where to deploy Kubernetes resources. Only default is supported at the moment.

Example: default

+ + + yugabyte_cloud + string + "dss" +

Cloud of yugabyte instances, used for partionning.

+

Should be set to dss unless you're doing advanced partitionning.

+ + + yugabyte_region + string + "uss-1" +

Region of yugabyte instances, used for partionning.

+

Should be different from others USS in a cluster.

+ + + yugabyte_zone + string + "zone" +

Zone of yugabyte instances, used for partionning.

+

Should be set to zone unless you're doing advanced partitionning.

+ + + yugabyte_light_resources + bool + false +

Enable light resources reservation for yugabyte instances.

+

Useful for a dev cluster when you don't want to overload your kubernetes cluster.

+ + + yugabyte_external_nodes + list(string) + [] +

Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. +Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"]

\ No newline at end of file diff --git a/deploy/infrastructure/modules/terraform-aws-dss/main.tf b/deploy/infrastructure/modules/terraform-aws-dss/main.tf index effdfae82..72fe00872 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/main.tf @@ -27,6 +27,7 @@ module "terraform-commons-dss" { authorization = var.authorization crdb_locality = var.crdb_locality crdb_external_nodes = var.crdb_external_nodes + node_count = var.node_count crdb_internal_nodes = module.terraform-aws-kubernetes.crdb_nodes ip_gateway = module.terraform-aws-kubernetes.ip_gateway kubernetes_api_endpoint = module.terraform-aws-kubernetes.kubernetes_api_endpoint diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf index b281fb6fe..c5fb86bd7 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf @@ -65,15 +65,23 @@ variable "crdb_hostname_suffix" { EOT } -variable "cluster_name" { +variable "datastore_type" { type = string description = <<-EOT - Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted) + Type of datastore used - Example: `dss-che-1` + Supported technologies: cockroachdb, yugabyte EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" } + variable "node_count" { type = number description = <<-EOT @@ -84,12 +92,21 @@ variable "node_count" { EOT validation { - condition = contains([1, 3], var.node_count) - error_message = "Currently, only 1 node or 3 nodes deployments are supported." + condition = (var.datastore_type == "cockroach" && contains([1, 3], var.node_count)) || (var.datastore_type == "yugabyte" && var.node_count > 0) + error_message = "Currently, only 1 node or 3 nodes deployments are supported for CockroachDB. If you use Yugabyte, you need to have at least one node." } } +variable "cluster_name" { + type = string + description = <<-EOT + Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted) + + Example: `dss-che-1` + EOT +} + variable "kubernetes_version" { type = string description = <<-EOT @@ -319,3 +336,62 @@ variable "kubernetes_namespace" { } } +variable "yugabyte_cloud" { + type = string + description = <<-EOT + Cloud of yugabyte instances, used for partionning. + + Should be set to dss unless you're doing advanced partitionning. + EOT + + default = "dss" +} + + +variable "yugabyte_region" { + type = string + description = <<-EOT + Region of yugabyte instances, used for partionning. + + Should be different from others USS in a cluster. + EOT + + default = "uss-1" +} + + +variable "yugabyte_zone" { + type = string + description = <<-EOT + Zone of yugabyte instances, used for partionning. + + Should be set to zone unless you're doing advanced partitionning. + EOT + + default = "zone" +} + + +variable "yugabyte_light_resources" { + type = bool + description = <<-EOT + Enable light resources reservation for yugabyte instances. + + Useful for a dev cluster when you don't want to overload your kubernetes cluster. + EOT + + default = false +} + + +variable "yugabyte_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. + Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"] + EOT + default = [] +} + + + diff --git a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md index 7249ece7b..2d66a5e28 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md @@ -61,11 +61,11 @@ For instance, if your CRDB nodes were addressable at 0.db.example.com,

Example: db.example.com

- cluster_name + datastore_type string - -

Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)

-

Example: dss-che-1

+ "cockroachdb" +

Type of datastore used

+

Supported technologies: cockroachdb, yugabyte

node_count @@ -74,6 +74,13 @@ For instance, if your CRDB nodes were addressable at 0.db.example.com,

Number of Kubernetes nodes which should correspond to the desired CockroachDB nodes. Currently, only single node or three nodes deployments are supported.

Example: 3

+ + + cluster_name + string + +

Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)

+

Example: dss-che-1

kubernetes_version @@ -239,6 +246,41 @@ Example: ["0.db.dss.example.com", "1.db.dss.example.com", &q "default"

Namespace where to deploy Kubernetes resources. Only default is supported at the moment.

Example: default

+ + + yugabyte_cloud + string + "dss" +

Cloud of yugabyte instances, used for partionning.

+

Should be set to dss unless you're doing advanced partitionning.

+ + + yugabyte_region + string + "uss-1" +

Region of yugabyte instances, used for partionning.

+

Should be different from others USS in a cluster.

+ + + yugabyte_zone + string + "zone" +

Zone of yugabyte instances, used for partionning.

+

Should be set to zone unless you're doing advanced partitionning.

+ + + yugabyte_light_resources + bool + false +

Enable light resources reservation for yugabyte instances.

+

Useful for a dev cluster when you don't want to overload your kubernetes cluster.

+ + + yugabyte_external_nodes + list(string) + [] +

Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. +Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"]

\ No newline at end of file diff --git a/deploy/infrastructure/modules/terraform-google-dss/main.tf b/deploy/infrastructure/modules/terraform-google-dss/main.tf index ea98a4dd6..19f459599 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/main.tf @@ -5,6 +5,7 @@ module "terraform-google-kubernetes" { google_zone = var.google_zone app_hostname = var.app_hostname crdb_hostname_suffix = var.crdb_hostname_suffix + datastore_type = var.datastore_type google_dns_managed_zone_name = var.google_dns_managed_zone_name google_machine_type = var.google_machine_type node_count = var.node_count @@ -15,25 +16,34 @@ module "terraform-google-kubernetes" { module "terraform-commons-dss" { # See variables.tf for variables description. - image = var.image - kubernetes_namespace = var.kubernetes_namespace - kubernetes_storage_class = var.google_kubernetes_storage_class - app_hostname = var.app_hostname - crdb_image_tag = var.crdb_image_tag - crdb_cluster_name = var.crdb_cluster_name - crdb_hostname_suffix = var.crdb_hostname_suffix - should_init = var.should_init - authorization = var.authorization - crdb_locality = var.crdb_locality - image_pull_secret = var.image_pull_secret - crdb_external_nodes = var.crdb_external_nodes - kubernetes_api_endpoint = module.terraform-google-kubernetes.kubernetes_api_endpoint - crdb_internal_nodes = module.terraform-google-kubernetes.crdb_nodes - ip_gateway = module.terraform-google-kubernetes.ip_gateway - ssl_policy = module.terraform-google-kubernetes.ssl_policy - kubernetes_cloud_provider_name = module.terraform-google-kubernetes.kubernetes_cloud_provider_name - kubernetes_context_name = module.terraform-google-kubernetes.kubernetes_context_name - kubernetes_get_credentials_cmd = module.terraform-google-kubernetes.kubernetes_get_credentials_cmd + image = var.image + kubernetes_namespace = var.kubernetes_namespace + kubernetes_storage_class = var.google_kubernetes_storage_class + app_hostname = var.app_hostname + crdb_image_tag = var.crdb_image_tag + crdb_cluster_name = var.crdb_cluster_name + crdb_hostname_suffix = var.crdb_hostname_suffix + datastore_type = var.datastore_type + should_init = var.should_init + authorization = var.authorization + crdb_locality = var.crdb_locality + image_pull_secret = var.image_pull_secret + crdb_external_nodes = var.crdb_external_nodes + yugabyte_cloud = var.yugabyte_cloud + yugabyte_region = var.yugabyte_region + yugabyte_zone = var.yugabyte_zone + yugabyte_light_resources = var.yugabyte_light_resources + yugabyte_external_nodes = var.yugabyte_external_nodes + node_count = var.node_count + kubernetes_api_endpoint = module.terraform-google-kubernetes.kubernetes_api_endpoint + crdb_internal_nodes = module.terraform-google-kubernetes.crdb_nodes + yugabyte_internal_masters_nodes = module.terraform-google-kubernetes.yugabyte_masters_nodes + yugabyte_internal_tservers_nodes = module.terraform-google-kubernetes.yugabyte_tservers_nodes + ip_gateway = module.terraform-google-kubernetes.ip_gateway + ssl_policy = module.terraform-google-kubernetes.ssl_policy + kubernetes_cloud_provider_name = module.terraform-google-kubernetes.kubernetes_cloud_provider_name + kubernetes_context_name = module.terraform-google-kubernetes.kubernetes_context_name + kubernetes_get_credentials_cmd = module.terraform-google-kubernetes.kubernetes_get_credentials_cmd source = "../../dependencies/terraform-commons-dss" } diff --git a/deploy/infrastructure/modules/terraform-google-dss/output.tf b/deploy/infrastructure/modules/terraform-google-dss/output.tf index 7be13880f..0f1486a3d 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/output.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/output.tf @@ -2,6 +2,14 @@ output "crdb_addresses" { value = module.terraform-google-kubernetes.crdb_addresses } +output "yugabyte_masters_addresses" { + value = module.terraform-google-kubernetes.yugabyte_masters_addresses +} + +output "yugabyte_tservers_addresses" { + value = module.terraform-google-kubernetes.yugabyte_tservers_addresses +} + output "gateway_address" { value = module.terraform-google-kubernetes.gateway_address } diff --git a/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars b/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars index 79ac8fb1f..cd8880f80 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars +++ b/deploy/infrastructure/modules/terraform-google-dss/terraform.dev.example.tfvars @@ -26,8 +26,14 @@ authorization = { } should_init = true +# Datastore +datastore_type = "cockroachdb" + # CockroachDB crdb_image_tag = "v24.1.3" crdb_cluster_name = "interuss-example" crdb_locality = "interuss_dss-dev-w6a" crdb_external_nodes = [] + +# Yugabyte +yugabyte_region = "uss-1" diff --git a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf index 94e9ee63f..5e43eb195 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf @@ -54,15 +54,23 @@ variable "crdb_hostname_suffix" { EOT } -variable "cluster_name" { +variable "datastore_type" { type = string description = <<-EOT - Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted) + Type of datastore used - Example: `dss-che-1` + Supported technologies: cockroachdb, yugabyte EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" } + variable "node_count" { type = number description = <<-EOT @@ -73,12 +81,21 @@ variable "node_count" { EOT validation { - condition = contains([1, 3], var.node_count) - error_message = "Currently, only 1 node or 3 nodes deployments are supported." + condition = (var.datastore_type == "cockroach" && contains([1, 3], var.node_count)) || (var.datastore_type == "yugabyte" && var.node_count > 0) + error_message = "Currently, only 1 node or 3 nodes deployments are supported for CockroachDB. If you use Yugabyte, you need to have at least one node." } } +variable "cluster_name" { + type = string + description = <<-EOT + Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted) + + Example: `dss-che-1` + EOT +} + variable "kubernetes_version" { type = string description = <<-EOT @@ -308,3 +325,62 @@ variable "kubernetes_namespace" { } } +variable "yugabyte_cloud" { + type = string + description = <<-EOT + Cloud of yugabyte instances, used for partionning. + + Should be set to dss unless you're doing advanced partitionning. + EOT + + default = "dss" +} + + +variable "yugabyte_region" { + type = string + description = <<-EOT + Region of yugabyte instances, used for partionning. + + Should be different from others USS in a cluster. + EOT + + default = "uss-1" +} + + +variable "yugabyte_zone" { + type = string + description = <<-EOT + Zone of yugabyte instances, used for partionning. + + Should be set to zone unless you're doing advanced partitionning. + EOT + + default = "zone" +} + + +variable "yugabyte_light_resources" { + type = bool + description = <<-EOT + Enable light resources reservation for yugabyte instances. + + Useful for a dev cluster when you don't want to overload your kubernetes cluster. + EOT + + default = false +} + + +variable "yugabyte_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. + Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"] + EOT + default = [] +} + + + diff --git a/deploy/infrastructure/utils/definitions/datastore_type.tf b/deploy/infrastructure/utils/definitions/datastore_type.tf new file mode 100644 index 000000000..b63a17237 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/datastore_type.tf @@ -0,0 +1,15 @@ +variable "datastore_type" { + type = string + description = <<-EOT + Type of datastore used + + Supported technologies: cockroachdb, yugabyte + EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" +} diff --git a/deploy/infrastructure/utils/definitions/node_count.tf b/deploy/infrastructure/utils/definitions/node_count.tf index 8d83ef002..513ba231d 100644 --- a/deploy/infrastructure/utils/definitions/node_count.tf +++ b/deploy/infrastructure/utils/definitions/node_count.tf @@ -8,7 +8,7 @@ variable "node_count" { EOT validation { - condition = contains([1, 3], var.node_count) - error_message = "Currently, only 1 node or 3 nodes deployments are supported." + condition = (var.datastore_type == "cockroach" && contains([1, 3], var.node_count)) || (var.datastore_type == "yugabyte" && var.node_count > 0) + error_message = "Currently, only 1 node or 3 nodes deployments are supported for CockroachDB. If you use Yugabyte, you need to have at least one node." } } diff --git a/deploy/infrastructure/utils/definitions/yugabyte_cloud.tf b/deploy/infrastructure/utils/definitions/yugabyte_cloud.tf new file mode 100644 index 000000000..51ef755d6 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_cloud.tf @@ -0,0 +1,10 @@ +variable "yugabyte_cloud" { + type = string + description = <<-EOT + Cloud of yugabyte instances, used for partionning. + + Should be set to dss unless you're doing advanced partitionning. + EOT + + default = "dss" +} diff --git a/deploy/infrastructure/utils/definitions/yugabyte_external_nodes.tf b/deploy/infrastructure/utils/definitions/yugabyte_external_nodes.tf new file mode 100644 index 000000000..fd065346d --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_external_nodes.tf @@ -0,0 +1,9 @@ +variable "yugabyte_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. + Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"] + EOT + default = [] +} + diff --git a/deploy/infrastructure/utils/definitions/yugabyte_light_resources.tf b/deploy/infrastructure/utils/definitions/yugabyte_light_resources.tf new file mode 100644 index 000000000..f776f7a79 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_light_resources.tf @@ -0,0 +1,10 @@ +variable "yugabyte_light_resources" { + type = bool + description = <<-EOT + Enable light resources reservation for yugabyte instances. + + Useful for a dev cluster when you don't want to overload your kubernetes cluster. + EOT + + default = false +} diff --git a/deploy/infrastructure/utils/definitions/yugabyte_region.tf b/deploy/infrastructure/utils/definitions/yugabyte_region.tf new file mode 100644 index 000000000..17613b9d1 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_region.tf @@ -0,0 +1,10 @@ +variable "yugabyte_region" { + type = string + description = <<-EOT + Region of yugabyte instances, used for partionning. + + Should be different from others USS in a cluster. + EOT + + default = "uss-1" +} diff --git a/deploy/infrastructure/utils/definitions/yugabyte_resources.tf b/deploy/infrastructure/utils/definitions/yugabyte_resources.tf new file mode 100644 index 000000000..abd429c92 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_resources.tf @@ -0,0 +1,11 @@ +variable "should_init" { + type = bool + description = <<-EOT + Set to false if joining an existing pool, true if creating the first DSS instance + for a pool. When set true, this can initialize the data directories on your cluster, + and prevent you from joining an existing pool. + + Example: `true` + EOT +} + diff --git a/deploy/infrastructure/utils/definitions/yugabyte_zone.tf b/deploy/infrastructure/utils/definitions/yugabyte_zone.tf new file mode 100644 index 000000000..735133f14 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/yugabyte_zone.tf @@ -0,0 +1,10 @@ +variable "yugabyte_zone" { + type = string + description = <<-EOT + Zone of yugabyte instances, used for partionning. + + Should be set to zone unless you're doing advanced partitionning. + EOT + + default = "zone" +} diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 779720041..ec17f32d8 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -23,7 +23,7 @@ # Variables per project # For all */terraform-* -GLOBAL_VARIABLES = ["app_hostname", "crdb_hostname_suffix"] +GLOBAL_VARIABLES = ["app_hostname", "crdb_hostname_suffix", "datastore_type", "node_count"] # dependencies/terraform-commons-dss COMMONS_DSS_VARIABLES = GLOBAL_VARIABLES + [ @@ -39,12 +39,16 @@ "crdb_locality", "crdb_external_nodes", "kubernetes_namespace", + "yugabyte_cloud", + "yugabyte_region", + "yugabyte_zone", + "yugabyte_light_resources", + "yugabyte_external_nodes", ] # dependencies/terraform-*-kubernetes COMMON_KUBERNETES_VARIABLES = GLOBAL_VARIABLES + [ "cluster_name", - "node_count", "kubernetes_version", ] diff --git a/deploy/operations/certificates-management/cluster.py b/deploy/operations/certificates-management/cluster.py index 00e918aa9..c65d376be 100644 --- a/deploy/operations/certificates-management/cluster.py +++ b/deploy/operations/certificates-management/cluster.py @@ -6,12 +6,13 @@ class Cluster(object): """Represent an instance of a cluster, expose paths""" - def __init__(self, name, cluster_context, namespace, organization, nodes_count): + def __init__(self, name, cluster_context, namespace, organization, nodes_count, nodes_public_address): self._name = name self.cluster_context = cluster_context self.namespace = namespace self.organization = organization self.nodes_count = nodes_count + self.nodes_public_address = nodes_public_address @property def name(self): @@ -113,6 +114,9 @@ def get_node_full_name_without_group(self, node_type, node_id): short_name = self.get_node_short_name(node_type, node_id) return f"{short_name}.{self.namespace}.svc.cluster.local" + def get_node_public_address(self, node_type, node_id): + return self.nodes_public_address.replace("", str(node_id)).replace("", node_type) + def get_node_cert_file(self, node_type, node_id): folder = getattr(self, f"{node_type}_certs_dir") full_name = self.get_node_full_name(node_type, node_id) @@ -123,6 +127,18 @@ def get_node_key_file(self, node_type, node_id): full_name = self.get_node_full_name(node_type, node_id) return f"{folder}/node.{full_name}.key" + def get_node_cert_second_file(self, node_type, node_id): + folder = getattr(self, f"{node_type}_certs_dir") + address = self.get_node_public_address(node_type, node_id) + if address: + return f"{folder}/node.{address}.crt" + + def get_node_key_second_file(self, node_type, node_id): + folder = getattr(self, f"{node_type}_certs_dir") + address = self.get_node_public_address(node_type, node_id) + if address: + return f"{folder}/node.{address}.key" + def get_node_csr_file(self, node_type, node_id): full_name = self.get_node_full_name(node_type, node_id) return f"{self.ca_key_dir}/node.{full_name}.csr" diff --git a/deploy/operations/certificates-management/dss-certs.py b/deploy/operations/certificates-management/dss-certs.py index ec12c868f..dc11a872f 100755 --- a/deploy/operations/certificates-management/dss-certs.py +++ b/deploy/operations/certificates-management/dss-certs.py @@ -48,6 +48,12 @@ def parse_args(): default="3", help="Number of yugabyte nodes in the cluster, default to 3", ) + parser.add_argument( + "--nodes-public-address", + metavar="NODES_PUBLIC_ADDRESS", + default="", + help="Public node address. Use to indicate id of the node (0, 1, ...), for the type (tserver, masters). Example: '..db.interuss.example'", + ) parser.add_argument( "--ca-file", metavar="CA_FILE", @@ -103,6 +109,7 @@ def main(): args.namespace, args.organization, args.nodes_count, + args.nodes_public_address, ) def read_input(): diff --git a/deploy/operations/certificates-management/nodes.py b/deploy/operations/certificates-management/nodes.py index 0a2e06c23..e787f2f04 100644 --- a/deploy/operations/certificates-management/nodes.py +++ b/deploy/operations/certificates-management/nodes.py @@ -1,6 +1,7 @@ import logging import subprocess import sys +import shutil from utils import get_cert_display_name @@ -15,6 +16,7 @@ def generate_node_config(cluster, node_type, node_id): full_name_without_group = cluster.get_node_full_name_without_group( node_type, node_id ) + public_address = cluster.get_node_public_address(node_type, node_id) with open(cluster.get_node_conf_file(node_type, node_id), "w") as f: f.write( @@ -28,9 +30,9 @@ def generate_node_config(cluster, node_type, node_id): # Multiple subject alternative names (SANs) such as IP Address, # DNS Name, Email, URI, and so on, can be specified under this section -[ req_ext] +[ req_ext ] subjectAltName = @alt_names -[alt_names] +[ alt_names ] DNS.1 = {short_name} DNS.2 = {full_name} DNS.3 = {short_name_group} @@ -41,6 +43,10 @@ def generate_node_config(cluster, node_type, node_id): """ ) + if public_address: + f.write(f"""DNS.8 = {public_address} +""") + l.info(f"Created {node_type} #{node_id} configuration file") @@ -48,16 +54,23 @@ def generate_node_key(cluster, node_type, node_id): l.debug(f"Generating {node_type} #{node_id} private key") + file = cluster.get_node_key_file(node_type, node_id) + subprocess.check_call( [ "openssl", "genrsa", "-out", - cluster.get_node_key_file(node_type, node_id), + file, "4096", ] ) + second_file = cluster.get_node_key_second_file(node_type, node_id) + + if second_file: + shutil.copy(file, second_file) + l.info(f"Generated {node_type} #{node_id} private key") @@ -87,6 +100,8 @@ def generate_node_cert(cluster, node_type, node_id): l.debug(f"Generating {node_type} #{node_id} certificate") + file = cluster.get_node_cert_file(node_type, node_id) + subprocess.check_call( [ "openssl", @@ -100,7 +115,7 @@ def generate_node_cert(cluster, node_type, node_id): "-policy", "my_policy", "-out", - cluster.get_node_cert_file(node_type, node_id), + file, "-outdir", getattr(cluster, f"{node_type}_certs_dir"), "-in", @@ -110,11 +125,18 @@ def generate_node_cert(cluster, node_type, node_id): "-batch", "-extfile", cluster.get_node_conf_file(node_type, node_id), + "-extensions", + "req_ext", ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) + second_file = cluster.get_node_cert_second_file(node_type, node_id) + + if second_file: + shutil.copy(file, second_file) + name = get_cert_display_name(cluster.get_node_cert_file(node_type, node_id)) l.info(f"Generated {node_type} #{node_id} certificate '{name}'") diff --git a/deploy/operations/ci/aws-1/variables.gen.tf b/deploy/operations/ci/aws-1/variables.gen.tf index b281fb6fe..c5fb86bd7 100644 --- a/deploy/operations/ci/aws-1/variables.gen.tf +++ b/deploy/operations/ci/aws-1/variables.gen.tf @@ -65,15 +65,23 @@ variable "crdb_hostname_suffix" { EOT } -variable "cluster_name" { +variable "datastore_type" { type = string description = <<-EOT - Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted) + Type of datastore used - Example: `dss-che-1` + Supported technologies: cockroachdb, yugabyte EOT + + validation { + condition = contains(["cockroachdb", "yugabyte"], var.datastore_type) + error_message = "Supported technologies: cockroachdb, yugabyte" + } + + default = "cockroachdb" } + variable "node_count" { type = number description = <<-EOT @@ -84,12 +92,21 @@ variable "node_count" { EOT validation { - condition = contains([1, 3], var.node_count) - error_message = "Currently, only 1 node or 3 nodes deployments are supported." + condition = (var.datastore_type == "cockroach" && contains([1, 3], var.node_count)) || (var.datastore_type == "yugabyte" && var.node_count > 0) + error_message = "Currently, only 1 node or 3 nodes deployments are supported for CockroachDB. If you use Yugabyte, you need to have at least one node." } } +variable "cluster_name" { + type = string + description = <<-EOT + Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted) + + Example: `dss-che-1` + EOT +} + variable "kubernetes_version" { type = string description = <<-EOT @@ -319,3 +336,62 @@ variable "kubernetes_namespace" { } } +variable "yugabyte_cloud" { + type = string + description = <<-EOT + Cloud of yugabyte instances, used for partionning. + + Should be set to dss unless you're doing advanced partitionning. + EOT + + default = "dss" +} + + +variable "yugabyte_region" { + type = string + description = <<-EOT + Region of yugabyte instances, used for partionning. + + Should be different from others USS in a cluster. + EOT + + default = "uss-1" +} + + +variable "yugabyte_zone" { + type = string + description = <<-EOT + Zone of yugabyte instances, used for partionning. + + Should be set to zone unless you're doing advanced partitionning. + EOT + + default = "zone" +} + + +variable "yugabyte_light_resources" { + type = bool + description = <<-EOT + Enable light resources reservation for yugabyte instances. + + Useful for a dev cluster when you don't want to overload your kubernetes cluster. + EOT + + default = false +} + + +variable "yugabyte_external_nodes" { + type = list(string) + description = <<-EOT + Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool. + Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"] + EOT + default = [] +} + + + diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index d5c8a1913..6521a0fbf 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -22,7 +22,7 @@ metadata: name: {{.Release.Name}}-core-service spec: minReadySeconds: 30 - replicas: {{ len .Values.loadBalancers.cockroachdbNodes }} + replicas: {{ if $.Values.cockroachdb.enabled -}}{{ len .Values.loadBalancers.cockroachdbNodes }}{{- else -}}{{ len .Values.loadBalancers.yugabyteMasterNodes }}{{- end }} selector: matchLabels: app: {{.Release.Name}}-core-service diff --git a/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml b/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml new file mode 100644 index 000000000..f3b47ce21 --- /dev/null +++ b/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml @@ -0,0 +1,92 @@ +{{- $cloudProvider := $.Values.global.cloudProvider}} + +{{- if $.Values.yugabyte.enabled }} + +# Master nodes Gateways +{{- range $i, $lb := .Values.loadBalancers.yugabyteMasterNodes }} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + {{- include (printf "%s-lb-crdb-annotations" $cloudProvider) + (dict + "name" (printf "%s-%s" "yugabyte-db-master-external-node" ( $i | toString) ) + "ip" $lb.ip + "subnet" $lb.subnet + "cloudProvider" $cloudProvider + ) | nindent 4 + }} + labels: + app: yugabyte + name: yugabyte-db-master-external-node-{{$i}} + name: yugabyte-db-master-external-node-{{$i}} +spec: + {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}} + ports: + - name: yugabyte-master-db-external-node-{{$i}} + port: 7100 + targetPort: 7100 + - name: yugabyte-master-ui-external-node-{{$i}} + port: 7000 + targetPort: 7000 + - name: yugabyte-master-ui2-external-node-{{$i}} + port: 9000 + targetPort: 9000 + publishNotReadyAddresses: true + selector: + statefulset.kubernetes.io/pod-name: yb-master-{{$i}} + type: LoadBalancer +{{- end }} + +# Tserver nodes Gateways +{{- range $i, $lb := .Values.loadBalancers.yugabyteTserverNodes }} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + {{- include (printf "%s-lb-crdb-annotations" $cloudProvider) + (dict + "name" (printf "%s-%s" " yugabyte-db-tserver-external-node" ( $i | toString) ) + "ip" $lb.ip + "subnet" $lb.subnet + "cloudProvider" $cloudProvider + ) | nindent 4 + }} + labels: + app: yugabyte + name: yugabyte-db-tserver-external-node-{{$i}} + name: yugabyte-db-tserver-external-node-{{$i}} +spec: + {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}} + ports: + - name: yugabyte-tserver-db-external-node-{{$i}} + port: 9100 + targetPort: 9100 + - name: yugabyte-tserver-ui-external-node-{{$i}} + port: 9000 + targetPort: 9000 + - name: yugabyte-tserver-ui2-external-node-{{$i}} + port: 7000 + targetPort: 7000 + - name: yugabyte-tserver-ycql-external-node-{{$i}} + port: 9042 + targetPort: 9042 + - name: yugabyte-tserver-ysql-external-node-{{$i}} + port: 5433 + targetPort: 5433 + - name: yugabyte-tserver-metrics-external-node-{{$i}} + port: 13000 + targetPort: 13000 + - name: yugabyte-tserver-metrics-2-external-node-{{$i}} + port: 12000 + targetPort: 12000 + publishNotReadyAddresses: true + selector: + statefulset.kubernetes.io/pod-name: yb-tserver-{{$i}} + type: LoadBalancer +{{- end }} +{{- end }} diff --git a/deploy/services/helm-charts/dss/values.example.yaml b/deploy/services/helm-charts/dss/values.example.yaml index 1e75f35f9..3725a31c5 100644 --- a/deploy/services/helm-charts/dss/values.example.yaml +++ b/deploy/services/helm-charts/dss/values.example.yaml @@ -61,6 +61,13 @@ loadBalancers: - ip: 192.168.0.22 # subnet: subnet-xxx # for aws only - ip: 192.168.0.23 +# subnet: subnet-xxx # for aws only + yugabyteMasterNodes: + - ip: 192.168.0.31 +# subnet: subnet-xxx # for aws only + - ip: 192.168.0.32 +# subnet: subnet-xxx # for aws only + - ip: 192.168.0.33 # subnet: subnet-xxx # for aws only dssGateway: ip: 192.168.0.10 diff --git a/deploy/services/helm-charts/dss/values.schema.json b/deploy/services/helm-charts/dss/values.schema.json index 8af475293..050ebdac5 100644 --- a/deploy/services/helm-charts/dss/values.schema.json +++ b/deploy/services/helm-charts/dss/values.schema.json @@ -21,7 +21,6 @@ } }, "required": [ - "tag" ] }, "fullnameOverride": { @@ -80,10 +79,6 @@ } }, "required": [ - "image", - "fullnameOverride", - "conf", - "statefulset", "enabled" ] }, @@ -184,6 +179,42 @@ ] } }, + "yugabyteMasterNodes": { + "type": "array", + "items": { + "properties": { + "ip": { + "description": "Public ip or cloud resource name.", + "type": "string" + }, + "subnet": { + "description": "Optional: Subnet of the public ip for AWS only", + "type": "string" + } + }, + "required": [ + "ip" + ] + } + }, + "yugabyteTserverNodes": { + "type": "array", + "items": { + "properties": { + "ip": { + "description": "Public ip or cloud resource name.", + "type": "string" + }, + "subnet": { + "description": "Optional: Subnet of the public ip for AWS only", + "type": "string" + } + }, + "required": [ + "ip" + ] + } + }, "dssGateway": { "type": "object", "properties": { @@ -260,8 +291,6 @@ } }, "required": [ - "cockroachdb", - "yugabyte", "loadBalancers", "dss", "global"