From 47442948a4e999a89fb4f5ddf473761d178625e1 Mon Sep 17 00:00:00 2001 From: Steven Dake Date: Thu, 15 Aug 2024 05:23:00 +0000 Subject: [PATCH 1/3] Introduce terraform Deploy Kubernetes using terraform. Of interest here is terraform.tfvars which can be customized to create a specific Kubernetes cluster. This implementation could be improved to set a Kubernetes configuration per line in a list, so that multiple Kubernetes deployments could be managed side-by-side. --- platform/terraform/00_init/README.md | 2 + platform/terraform/00_init/keygen.sh | 13 +++ platform/terraform/01_compartment/README.md | 27 +++++ platform/terraform/02_kubernetes/README.md | 33 ++++++ platform/terraform/02_kubernetes/locals.tf | 70 ++++++++++++ platform/terraform/02_kubernetes/main.tf | 54 ++++++++++ platform/terraform/02_kubernetes/outputs.tf | 3 + platform/terraform/02_kubernetes/providers.tf | 13 +++ .../terraform/02_kubernetes/terraform.tfvars | 35 ++++++ platform/terraform/02_kubernetes/variables.tf | 101 ++++++++++++++++++ platform/terraform/02_kubernetes/versions.tf | 8 ++ 11 files changed, 359 insertions(+) create mode 100644 platform/terraform/00_init/README.md create mode 100644 platform/terraform/00_init/keygen.sh create mode 100644 platform/terraform/01_compartment/README.md create mode 100644 platform/terraform/02_kubernetes/README.md create mode 100644 platform/terraform/02_kubernetes/locals.tf create mode 100644 platform/terraform/02_kubernetes/main.tf create mode 100644 platform/terraform/02_kubernetes/outputs.tf create mode 100644 platform/terraform/02_kubernetes/providers.tf create mode 100755 platform/terraform/02_kubernetes/terraform.tfvars create mode 100644 platform/terraform/02_kubernetes/variables.tf create mode 100644 platform/terraform/02_kubernetes/versions.tf diff --git a/platform/terraform/00_init/README.md b/platform/terraform/00_init/README.md new file mode 100644 index 0000000..48e3f8c --- /dev/null +++ b/platform/terraform/00_init/README.md @@ -0,0 +1,2 @@ +# Configure OCI + diff --git a/platform/terraform/00_init/keygen.sh b/platform/terraform/00_init/keygen.sh new file mode 100644 index 0000000..671e338 --- /dev/null +++ b/platform/terraform/00_init/keygen.sh @@ -0,0 +1,13 @@ +# https://docs.oracle.com/en-us/iaas/developer-tutorials/tutorials/tf-provider/01-summary.htm +mkdir $HOME/.oci +chmod 700 $HOME/.oci +openssl genrsa -out $HOME/.oci/steve_private.pem 2048 +chmod 600 $HOME/.oci/steve_private.pem +openssl rsa -pubout -in $HOME/.oci/steve_private.pem -out $HOME/.oci/steve_public.pem +cat $HOME/.oci/steve_public.pem + +# Configure OCI cloud +oci setup config + +# manually copy config variables to 01_kubernetes/terraform.tfvars +echo configure 01_kubernetes/terraform.tfvars diff --git a/platform/terraform/01_compartment/README.md b/platform/terraform/01_compartment/README.md new file mode 100644 index 0000000..a36baff --- /dev/null +++ b/platform/terraform/01_compartment/README.md @@ -0,0 +1,27 @@ +# Using compartments + +Compartments are the fundamental IAM feature used by Oracle Cloud to isolate cloud resources. Compartments +can have children in a tree structure, or be simpler with a depth of one. We use the `engineering` compartment +to house all engineering work. + +Anything in production is housed in `production`. + +During development, compartments are an awesome way to start over. Simply delete the compartment you were working +in, and all resources within the compartment are deleted. + +# Create a compartmnet + +This will create a compartment for you within the `engineering`. + +```console +parent_compartment=$(oci iam compartment list --query 'data[?name==`"engineering"`].{compartment_id: "id"}' --output json | jq -r '.[0].compartment_id') +oci iam compartment create --name $(whoami)-$(date +%Y%m%d) --description "development compartment" --compartment-id ${parent_compartment} +``` + +# Delete a compartment + +Replace the OCID with the compartment id you wish to delete. Never delete `engineering`. + +```console +oci iam compartment delete --compartment-id ocid1.compartment.oc1..aaaaaaaayh4wcewcyj4ns3no4eu6eyfwj3ncaexs73mz2c35cfdwv4xfeejq +``` diff --git a/platform/terraform/02_kubernetes/README.md b/platform/terraform/02_kubernetes/README.md new file mode 100644 index 0000000..8b76410 --- /dev/null +++ b/platform/terraform/02_kubernetes/README.md @@ -0,0 +1,33 @@ + +# Deploy and use Kubernetes + +Initialize terraform: + +``` +terraform init +``` + +Create a Kubernetes deployment: + +``` +terraform apply +``` + +Destroy the Kubernetes deployment: + +``` +terraform destroy +``` + +When you create the Kubernetes deployment, an `ocid.cluster....` is printed. + +Set the OCID cluster enviornment variable: +``` +CLUSTER_OCID="value from terraform apply" +``` + +Setup `kubectl` via `$HOME/.kube/config` + +``` +oci ce cluster create-kubeconfig --cluster-id "${CLUSTER_OCID}" --file $HOME/.kube/config --region us-phoenix-1 --token-version 2.0.0 --kube-endpoint PUBLIC_ENDPOINT +``` diff --git a/platform/terraform/02_kubernetes/locals.tf b/platform/terraform/02_kubernetes/locals.tf new file mode 100644 index 0000000..2edc0e6 --- /dev/null +++ b/platform/terraform/02_kubernetes/locals.tf @@ -0,0 +1,70 @@ +locals { + + ### + # + # Networking Configuration + + cni_type = "flannel" + kubeproxy_mode = "iptables" + create_drg = true + + + ### + # + # Kubernetes Control Plane Configuration + + create_cluster = true + cluster_type = "basic" + oke_control_plane = "public" + control_plane_allowed_cidrs = ["0.0.0.0/0"] + control_plane_is_public = true + assign_public_ip_to_control_plane = true + create_iam_resources = true + + + ### + # + # Kubernetes Worker Nodes Configuration + + worker_image_type = "oke" + worker_pool_mode = "node-pool" + allow_worker_ssh_access = false + worker_pools = { + np1 = { + create = true, + size = var.worker_nodes, + shape = "VM.Standard.E4.Flex", + ocpus = var.worker_cpu, + memory = var.worker_memory, + } + } + + worker_cloud_init = [ + { + content = <<-EOT + runcmd: + - 'echo "Kernel module configuration for Istio and worker node initialization"' + - 'modprobe br_netfilter' + - 'modprobe nf_nat' + - 'modprobe xt_REDIRECT' + - 'modprobe xt_owner' + - 'modprobe iptable_nat' + - 'modprobe iptable_mangle' + - 'modprobe iptable_filter' + - '/usr/libexec/oci-growfs -y' + - 'timedatectl set-timezone Australia/Sydney' + - 'curl --fail -H "Authorization: Bearer Oracle" -L0 http://169.254.169.254/opc/v2/instance/metadata/oke_init_script | base64 --decode >/var/run/oke-init.sh' + - 'bash -x /var/run/oke-init.sh' + EOT + content_type = "text/cloud-config", + } + ] + + ### + # + # Extras + + create_bastion = false + create_service_account = true + create_operator = false +} diff --git a/platform/terraform/02_kubernetes/main.tf b/platform/terraform/02_kubernetes/main.tf new file mode 100644 index 0000000..b1c2961 --- /dev/null +++ b/platform/terraform/02_kubernetes/main.tf @@ -0,0 +1,54 @@ +module "kubernetes" { + source = "git::https://github.com/oracle-terraform-modules/terraform-oci-oke" + providers = { + oci.home = oci + } + + api_fingerprint = var.api_fingerprint + api_private_key_path = var.api_private_key_path + tenancy_id = var.tenancy_id + compartment_id = var.compartment_id + user_id = var.user_id + vcn_cidrs = var.vcn_cidrs + region = var.region + kubernetes_version = var.kubernetes_version + pods_cidr = var.pods_cidr + services_cidr = var.services_cidr + + ### + # + # Networking Configuration + + cni_type = local.cni_type + kubeproxy_mode = local.kubeproxy_mode + create_drg = local.create_drg + + + ### + # + # Kubernetes Control Plane Configuration + + create_cluster = local.create_cluster + cluster_type = local.cluster_type + control_plane_allowed_cidrs = local.control_plane_allowed_cidrs + control_plane_is_public = local.control_plane_is_public + assign_public_ip_to_control_plane = local.assign_public_ip_to_control_plane + #create_iam_resources = local.create_iam_resources + + ### + # + # Kubernetes Worker Nodes Configuration + + create_iam_resources = local.create_iam_resources + worker_pool_mode = local.worker_pool_mode + allow_worker_ssh_access = local.allow_worker_ssh_access + worker_pools = local.worker_pools + worker_cloud_init = local.worker_cloud_init + + ### + # + # Extra nodes + + create_bastion = local.create_bastion + create_operator = local.create_operator +} diff --git a/platform/terraform/02_kubernetes/outputs.tf b/platform/terraform/02_kubernetes/outputs.tf new file mode 100644 index 0000000..8997089 --- /dev/null +++ b/platform/terraform/02_kubernetes/outputs.tf @@ -0,0 +1,3 @@ +output "cluster_ocid" { + value = module.kubernetes.cluster_id +} diff --git a/platform/terraform/02_kubernetes/providers.tf b/platform/terraform/02_kubernetes/providers.tf new file mode 100644 index 0000000..23fb39b --- /dev/null +++ b/platform/terraform/02_kubernetes/providers.tf @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +provider "oci" { + fingerprint = var.api_fingerprint + private_key_path = var.api_private_key_path + region = var.region + compartment_ocid = var.compartment_id + tenancy_ocid = var.tenancy_id + user_ocid = var.user_id + alias = "home" + ignore_defined_tags = ["Oracle-Tags.CreatedBy", "Oracle-Tags.CreatedOn"] +} diff --git a/platform/terraform/02_kubernetes/terraform.tfvars b/platform/terraform/02_kubernetes/terraform.tfvars new file mode 100755 index 0000000..08aa6a7 --- /dev/null +++ b/platform/terraform/02_kubernetes/terraform.tfvars @@ -0,0 +1,35 @@ +### +# +# Authentication + +api_fingerprint = "c6:1d:a5:54:c2:d2:67:26:6a:81:b8:e0:d2:f5:f7:a7" +api_private_key_path = "/hoem/sdake/.oci/oci_api_key.pem" + + +### +# +# Identity Management + +tenancy_id = "ocid1.tenancy.oc1..aaaaaaaa6vyjrctvv5ax3lzuah3ldtlnrvni6hxcqdzcfoxjw5stgu4vz32q" +compartment_id = "ocid1.compartment.oc1..aaaaaaaaq6xqdldlmtkmkpypkhsjymplonmuvbfpdqfii7ezu6b23utwqtba" +user_id = "ocid1.user.oc1..aaaaaaaa64i4tqgymgevje33u6tx7ejxgh2dipggg42lwikdr4f2ouwids5a" + + +### +# +# Networking Configuration + +vcn_cidrs = ["10.1.0.0/16"] + + +### +# +# Kubernetes Configuration + +region = "us-phoenix-1" +kubernetes_version = "v1.30.1" +worker_nodes = 2 +worker_cpu = 2 +worker_memory = 8 +pods_cidr = "10.201.0.0/16" +services_cidr = "10.101.0.0/16" diff --git a/platform/terraform/02_kubernetes/variables.tf b/platform/terraform/02_kubernetes/variables.tf new file mode 100644 index 0000000..51a9d92 --- /dev/null +++ b/platform/terraform/02_kubernetes/variables.tf @@ -0,0 +1,101 @@ +### +# +# Authentication + +variable "api_fingerprint" { + description = "Fingerprint of the API private key to use with OCI API." + type = string +} + +variable "api_private_key_path" { + description = "The path to the OCI API private key pem file." + type = string +} + +#variable "ssh_private_key_path" { +# description = "The path to ssh private key." +# type = string +#} + +#variable "ssh_public_key_path" { +# description = "The path to ssh public key." +# type = string +#} + +### +# +# Identity Management + +variable "user_id" { + description = "The id of the user that Terraform will use to create the resources." + type = string +} +variable "tenancy_id" { + description = "The tenancy id of the OCI Cloud Account in which to create the resources." + type = string +} + +variable "compartment_id" { + description = "The compartment id where to create all resources." + type = string +} + +# this may not be needed +#variable "home_region" { +# description = "The home region for this compartment." +# type = string +#} + + +### +# +# Cluster Networking Configuration + +# It would be cool to enhance this such that an array of clusters could be created. +variable "vcn_cidrs" { + description = "VCN CIDRs. I don't know how this is used" + type = list +} +variable "pods_cidr" { + description = "Network CIDR associated with PODs. Must be a /16 that does not overlap with other networks." + type = string +} + +variable "services_cidr" { + description = "Services CIDR associated with Services. Must be a /16 tha does not overlap with other networks." + type = string +} + +# It would be cool to enhance this such that an array of clusters could be created. +### +# +# Kubernetes Control Plane Configuration + +variable "region" { + description = "Create Kubernetes in this region." + type = string +} + +variable "kubernetes_version" { + default = "v1.30.1" + description = "Create Kubernetes using this version." + type = string +} + +variable "worker_nodes" { + default = "3" + description = "Create Kubernetes with this worker node count." + type = number +} + +variable "worker_memory" { + default = "16" + description = "Create each worker with this much memory in gigabytes." + type = number +} + +variable "worker_cpu" { + default = "4" + description = "Create each worker with this many virtual CPUs." + type = number +} diff --git a/platform/terraform/02_kubernetes/versions.tf b/platform/terraform/02_kubernetes/versions.tf new file mode 100644 index 0000000..75019aa --- /dev/null +++ b/platform/terraform/02_kubernetes/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + oci = { + source = "oracle/oci" + } + } + required_version = ">= 1.0.0" +} From 98be4df98d6f6c74c2f9e5c1d1c53c218c0e2f4b Mon Sep 17 00:00:00 2001 From: Steven Dake Date: Thu, 15 Aug 2024 05:28:40 +0000 Subject: [PATCH 2/3] Remove unnecessary variables Neither a bastion nor operator host is created. --- platform/terraform/02_kubernetes/variables.tf | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/platform/terraform/02_kubernetes/variables.tf b/platform/terraform/02_kubernetes/variables.tf index 51a9d92..0b15a3c 100644 --- a/platform/terraform/02_kubernetes/variables.tf +++ b/platform/terraform/02_kubernetes/variables.tf @@ -12,15 +12,6 @@ variable "api_private_key_path" { type = string } -#variable "ssh_private_key_path" { -# description = "The path to ssh private key." -# type = string -#} - -#variable "ssh_public_key_path" { -# description = "The path to ssh public key." -# type = string -#} ### # @@ -40,12 +31,6 @@ variable "compartment_id" { type = string } -# this may not be needed -#variable "home_region" { -# description = "The home region for this compartment." -# type = string -#} - ### # @@ -66,10 +51,11 @@ variable "services_cidr" { type = string } -# It would be cool to enhance this such that an array of clusters could be created. + ### # # Kubernetes Control Plane Configuration +# It would be cool to enhance this such that an array of clusters could be created. variable "region" { description = "Create Kubernetes in this region." From 9a3aad23134c528950794007751db69ad403b1d3 Mon Sep 17 00:00:00 2001 From: Steven Dake Date: Thu, 15 Aug 2024 06:05:30 +0000 Subject: [PATCH 3/3] Remove vcn_cidrs and increase worker capacity --- platform/terraform/02_kubernetes/main.tf | 1 - platform/terraform/02_kubernetes/terraform.tfvars | 11 ++--------- platform/terraform/02_kubernetes/variables.tf | 5 +---- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/platform/terraform/02_kubernetes/main.tf b/platform/terraform/02_kubernetes/main.tf index b1c2961..3955579 100644 --- a/platform/terraform/02_kubernetes/main.tf +++ b/platform/terraform/02_kubernetes/main.tf @@ -9,7 +9,6 @@ module "kubernetes" { tenancy_id = var.tenancy_id compartment_id = var.compartment_id user_id = var.user_id - vcn_cidrs = var.vcn_cidrs region = var.region kubernetes_version = var.kubernetes_version pods_cidr = var.pods_cidr diff --git a/platform/terraform/02_kubernetes/terraform.tfvars b/platform/terraform/02_kubernetes/terraform.tfvars index 08aa6a7..8ccca5d 100755 --- a/platform/terraform/02_kubernetes/terraform.tfvars +++ b/platform/terraform/02_kubernetes/terraform.tfvars @@ -15,21 +15,14 @@ compartment_id = "ocid1.compartment.oc1..aaaaaaaaq6xqdldlmtkmkpypkhsjymplonmuvbf user_id = "ocid1.user.oc1..aaaaaaaa64i4tqgymgevje33u6tx7ejxgh2dipggg42lwikdr4f2ouwids5a" -### -# -# Networking Configuration - -vcn_cidrs = ["10.1.0.0/16"] - - ### # # Kubernetes Configuration region = "us-phoenix-1" kubernetes_version = "v1.30.1" -worker_nodes = 2 +worker_nodes = 3 worker_cpu = 2 -worker_memory = 8 +worker_memory = 16 pods_cidr = "10.201.0.0/16" services_cidr = "10.101.0.0/16" diff --git a/platform/terraform/02_kubernetes/variables.tf b/platform/terraform/02_kubernetes/variables.tf index 0b15a3c..ab3be85 100644 --- a/platform/terraform/02_kubernetes/variables.tf +++ b/platform/terraform/02_kubernetes/variables.tf @@ -37,10 +37,7 @@ variable "compartment_id" { # Cluster Networking Configuration # It would be cool to enhance this such that an array of clusters could be created. -variable "vcn_cidrs" { - description = "VCN CIDRs. I don't know how this is used" - type = list -} + variable "pods_cidr" { description = "Network CIDR associated with PODs. Must be a /16 that does not overlap with other networks." type = string