From 9dba15d4b0ba53ff5819d47023d8374295e54bc4 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 17 Jun 2025 16:16:23 +0200 Subject: [PATCH] Yugaybte in AWS --- .../terraform-aws-kubernetes/network_dns.tf | 26 +++++++++- .../terraform-aws-kubernetes/network_lb.tf | 26 +++++++++- .../terraform-aws-kubernetes/output.tf | 34 ++++++++++++- .../terraform-commons-dss/helm.tf | 13 +++++ .../modules/terraform-aws-dss/main.tf | 51 +++++++++++-------- .../modules/terraform-aws-dss/output.tf | 8 +++ .../dss/templates/yugabyte-loadbalancers.yaml | 32 ++++++------ 7 files changed, 150 insertions(+), 40 deletions(-) diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_dns.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_dns.tf index 355cb8f99..1dde37191 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_dns.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_dns.tf @@ -1,6 +1,8 @@ locals { - crdb_hostnames = var.aws_route53_zone_id == "" ? {} : { for i in aws_eip.ip_crdb[*] : i.tags.ExpectedDNS => i.public_ip } + crdb_hostnames = var.aws_route53_zone_id == "" ? {} : { for i in aws_eip.ip_crdb[*] : i.tags.ExpectedDNS => i.public_ip } + yugabyte_master_hostnames = var.aws_route53_zone_id == "" ? {} : { for i in aws_eip.ip_yugabyte_masters[*] : i.tags.ExpectedDNS => i.public_ip } + yugabyte_tserver_hostnames = var.aws_route53_zone_id == "" ? {} : { for i in aws_eip.ip_yugabyte_tservers[*] : i.tags.ExpectedDNS => i.public_ip } } @@ -37,3 +39,25 @@ resource "aws_route53_record" "crdb_hostname" { ttl = 300 records = [each.value] } + +# Yugabyte master nodes DNS +resource "aws_route53_record" "yugabyte_master_hostnames" { + for_each = local.yugabyte_master_hostnames + + zone_id = var.aws_route53_zone_id + name = each.key + type = "A" + ttl = 300 + records = [each.value] +} + +# Yugabyte tserver nodes DNS +resource "aws_route53_record" "yugabyte_tserver_hostnames" { + for_each = local.yugabyte_tserver_hostnames + + zone_id = var.aws_route53_zone_id + name = each.key + type = "A" + ttl = 300 + records = [each.value] +} diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf index b436a49c7..1ccd8bcda 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf @@ -73,7 +73,7 @@ resource "aws_eip" "gateway" { # Public Elastic IPs for the crdb instances resource "aws_eip" "ip_crdb" { - count = var.node_count + count = var.datastore_type == "cockroachdb" ? var.node_count : 0 vpc = true tags = { @@ -82,3 +82,27 @@ resource "aws_eip" "ip_crdb" { ExpectedDNS = format("%s.%s", count.index, var.crdb_hostname_suffix) } } + +# Public Elastic IPs for the yubagybte master instances +resource "aws_eip" "ip_yugabyte_masters" { + count = var.datastore_type == "yugabyte" ? var.node_count : 0 + vpc = true + + tags = { + Name = format("%s-ip-yugabyte-master%v", var.cluster_name, count.index) + # Preserve mapping between ips and hostnames + ExpectedDNS = format("%s.master.%s", count.index, var.crdb_hostname_suffix) + } +} + +# Public Elastic IPs for the yubagybte tserver instances +resource "aws_eip" "ip_yugabyte_tservers" { + count = var.datastore_type == "yugabyte" ? var.node_count : 0 + vpc = true + + tags = { + Name = format("%s-ip-yugabyte-tserver%v", var.cluster_name, count.index) + # Preserve mapping between ips and hostnames + ExpectedDNS = format("%s.tserver.%s", count.index, var.crdb_hostname_suffix) + } +} diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf index a6d238d77..e7bf3dc25 100644 --- a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf +++ b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf @@ -30,10 +30,42 @@ output "crdb_nodes" { ] } +output "yugabyte_masters_nodes" { + value = [ + for i in aws_eip.ip_yugabyte_masters : { + ip = i.allocation_id + dns = i.tags.ExpectedDNS + } + ] + depends_on = [ + aws_eip.ip_yugabyte_masters + ] +} + +output "yugabyte_tservers_nodes" { + value = [ + for i in aws_eip.ip_yugabyte_tservers : { + ip = i.allocation_id + dns = i.tags.ExpectedDNS + } + ] + depends_on = [ + aws_eip.ip_yugabyte_tservers + ] +} + output "crdb_addresses" { value = [for i in aws_eip.ip_crdb[*] : { expected_dns : i.tags.ExpectedDNS, address : i.public_ip }] } +output "yugabyte_masters_addresses" { + value = [for i in aws_eip.ip_yugabyte_masters[*] : { expected_dns : i.tags.ExpectedDNS, address : i.public_ip }] +} + +output "yugabyte_tservers_addresses" { + value = [for i in aws_eip.ip_yugabyte_tservers[*] : { expected_dns : i.tags.ExpectedDNS, address : i.public_ip }] +} + output "gateway_address" { value = { expected_dns : aws_eip.gateway[0].tags.ExpectedDNS, @@ -56,4 +88,4 @@ output "workload_subnet" { output "iam_role_node_group_arn" { value = aws_iam_role.dss-cluster-node-group.arn -} \ No newline at end of file +} diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf index d799f1f0c..95e8f80c9 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf @@ -102,6 +102,19 @@ resource "local_file" "helm_chart_values" { totalMasters = length(var.yugabyte_external_nodes) + var.node_count } + storage = { + master = { + storageClass = var.kubernetes_storage_class + } + tserver = { + storageClass = var.kubernetes_storage_class + } + } + + preflight = { + skipUlimit = true + } + master = { extraEnv = [{ name = "HOSTNAMENO" diff --git a/deploy/infrastructure/modules/terraform-aws-dss/main.tf b/deploy/infrastructure/modules/terraform-aws-dss/main.tf index 72fe00872..44162ba27 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/main.tf @@ -4,6 +4,7 @@ module "terraform-aws-kubernetes" { aws_region = var.aws_region app_hostname = var.app_hostname crdb_hostname_suffix = var.crdb_hostname_suffix + datastore_type = var.datastore_type aws_instance_type = var.aws_instance_type aws_route53_zone_id = var.aws_route53_zone_id aws_iam_permissions_boundary = var.aws_iam_permissions_boundary @@ -15,27 +16,35 @@ module "terraform-aws-kubernetes" { module "terraform-commons-dss" { # See variables.tf for variables description. - image = var.image - image_pull_secret = var.image_pull_secret - kubernetes_namespace = var.kubernetes_namespace - kubernetes_storage_class = var.aws_kubernetes_storage_class - app_hostname = var.app_hostname - crdb_image_tag = var.crdb_image_tag - crdb_cluster_name = var.crdb_cluster_name - crdb_hostname_suffix = var.crdb_hostname_suffix - should_init = var.should_init - authorization = var.authorization - crdb_locality = var.crdb_locality - crdb_external_nodes = var.crdb_external_nodes - node_count = var.node_count - crdb_internal_nodes = module.terraform-aws-kubernetes.crdb_nodes - ip_gateway = module.terraform-aws-kubernetes.ip_gateway - kubernetes_api_endpoint = module.terraform-aws-kubernetes.kubernetes_api_endpoint - kubernetes_cloud_provider_name = module.terraform-aws-kubernetes.kubernetes_cloud_provider_name - kubernetes_context_name = module.terraform-aws-kubernetes.kubernetes_context_name - kubernetes_get_credentials_cmd = module.terraform-aws-kubernetes.kubernetes_get_credentials_cmd - workload_subnet = module.terraform-aws-kubernetes.workload_subnet - gateway_cert_name = module.terraform-aws-kubernetes.app_hostname_cert_arn + image = var.image + image_pull_secret = var.image_pull_secret + kubernetes_namespace = var.kubernetes_namespace + kubernetes_storage_class = var.aws_kubernetes_storage_class + app_hostname = var.app_hostname + crdb_image_tag = var.crdb_image_tag + crdb_cluster_name = var.crdb_cluster_name + crdb_hostname_suffix = var.crdb_hostname_suffix + datastore_type = var.datastore_type + should_init = var.should_init + authorization = var.authorization + crdb_locality = var.crdb_locality + crdb_external_nodes = var.crdb_external_nodes + node_count = var.node_count + yugabyte_cloud = var.yugabyte_cloud + yugabyte_region = var.yugabyte_region + yugabyte_zone = var.yugabyte_zone + yugabyte_light_resources = var.yugabyte_light_resources + yugabyte_external_nodes = var.yugabyte_external_nodes + crdb_internal_nodes = module.terraform-aws-kubernetes.crdb_nodes + yugabyte_internal_masters_nodes = module.terraform-aws-kubernetes.yugabyte_masters_nodes + yugabyte_internal_tservers_nodes = module.terraform-aws-kubernetes.yugabyte_tservers_nodes + ip_gateway = module.terraform-aws-kubernetes.ip_gateway + kubernetes_api_endpoint = module.terraform-aws-kubernetes.kubernetes_api_endpoint + kubernetes_cloud_provider_name = module.terraform-aws-kubernetes.kubernetes_cloud_provider_name + kubernetes_context_name = module.terraform-aws-kubernetes.kubernetes_context_name + kubernetes_get_credentials_cmd = module.terraform-aws-kubernetes.kubernetes_get_credentials_cmd + workload_subnet = module.terraform-aws-kubernetes.workload_subnet + gateway_cert_name = module.terraform-aws-kubernetes.app_hostname_cert_arn source = "../../dependencies/terraform-commons-dss" } diff --git a/deploy/infrastructure/modules/terraform-aws-dss/output.tf b/deploy/infrastructure/modules/terraform-aws-dss/output.tf index 4ab65ab50..9d282a99f 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/output.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/output.tf @@ -2,6 +2,14 @@ output "crdb_addresses" { value = module.terraform-aws-kubernetes.crdb_addresses } +output "yugabyte_masters_addresses" { + value = module.terraform-aws-kubernetes.yugabyte_masters_addresses +} + +output "yugabyte_tservers_addresses" { + value = module.terraform-aws-kubernetes.yugabyte_tservers_addresses +} + output "gateway_address" { value = module.terraform-aws-kubernetes.gateway_address } diff --git a/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml b/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml index f3b47ce21..079341b95 100644 --- a/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml +++ b/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml @@ -12,7 +12,7 @@ metadata: service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" {{- include (printf "%s-lb-crdb-annotations" $cloudProvider) (dict - "name" (printf "%s-%s" "yugabyte-db-master-external-node" ( $i | toString) ) + "name" (printf "%s-%s" "ybdb-master-ext" ( $i | toString) ) "ip" $lb.ip "subnet" $lb.subnet "cloudProvider" $cloudProvider @@ -20,18 +20,18 @@ metadata: }} labels: app: yugabyte - name: yugabyte-db-master-external-node-{{$i}} - name: yugabyte-db-master-external-node-{{$i}} + name: ybdb-master-ext-{{$i}} + name: ybdb-master-ext-{{$i}} spec: {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}} ports: - - name: yugabyte-master-db-external-node-{{$i}} + - name: yugabyte-master-db-ext-{{$i}} port: 7100 targetPort: 7100 - - name: yugabyte-master-ui-external-node-{{$i}} + - name: yugabyte-master-ui-ext-{{$i}} port: 7000 targetPort: 7000 - - name: yugabyte-master-ui2-external-node-{{$i}} + - name: yugabyte-master-ui2-ext-{{$i}} port: 9000 targetPort: 9000 publishNotReadyAddresses: true @@ -50,7 +50,7 @@ metadata: service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" {{- include (printf "%s-lb-crdb-annotations" $cloudProvider) (dict - "name" (printf "%s-%s" " yugabyte-db-tserver-external-node" ( $i | toString) ) + "name" (printf "%s-%s" " ybdb-tserver-ext" ( $i | toString) ) "ip" $lb.ip "subnet" $lb.subnet "cloudProvider" $cloudProvider @@ -58,30 +58,30 @@ metadata: }} labels: app: yugabyte - name: yugabyte-db-tserver-external-node-{{$i}} - name: yugabyte-db-tserver-external-node-{{$i}} + name: yuga-tserver-ext-{{$i}} + name: yuga-tserver-ext-{{$i}} spec: {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}} ports: - - name: yugabyte-tserver-db-external-node-{{$i}} + - name: yugabyte-tserver-db-ext-{{$i}} port: 9100 targetPort: 9100 - - name: yugabyte-tserver-ui-external-node-{{$i}} + - name: yugabyte-tserver-ui-ext-{{$i}} port: 9000 targetPort: 9000 - - name: yugabyte-tserver-ui2-external-node-{{$i}} + - name: yugabyte-tserver-ui2-ext-{{$i}} port: 7000 targetPort: 7000 - - name: yugabyte-tserver-ycql-external-node-{{$i}} + - name: yugabyte-tserver-ycql-ext-{{$i}} port: 9042 targetPort: 9042 - - name: yugabyte-tserver-ysql-external-node-{{$i}} + - name: yugabyte-tserver-ysql-ext-{{$i}} port: 5433 targetPort: 5433 - - name: yugabyte-tserver-metrics-external-node-{{$i}} + - name: yugabyte-tserver-metrics-ext-{{$i}} port: 13000 targetPort: 13000 - - name: yugabyte-tserver-metrics-2-external-node-{{$i}} + - name: yugabyte-tserver-metrics-2-ext-{{$i}} port: 12000 targetPort: 12000 publishNotReadyAddresses: true