diff --git a/.gitignore b/.gitignore index 498c22b..f9320f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ -myazure.publishsettings -*.env .DS_Store *~ *.swp @@ -9,4 +7,9 @@ myazure.publishsettings *.zip *.pyc *.dist-info/ -api/_tes* \ No newline at end of file +data/ +terraform.tfstate +terraform.tfstate.backup +tmp/ +*.env +.terraform/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..61f9f99 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,16 @@ +# This file is a template, and might need editing before it works on your project. +# Official docker image. +image: docker:latest + +services: + - docker:dind + +build: + stage: build + tags: + - foo + script: + - export IMAGE_TAG=$(echo -en $CI_COMMIT_REF_NAME | tr -c '[:alnum:]_.-' '-') + - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY + - docker build --pull -t "$CI_REGISTRY_IMAGE:$IMAGE_TAG" ./provisioning + - docker push "$CI_REGISTRY_IMAGE:$IMAGE_TAG" diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 752b71c..0000000 --- a/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM golang:alpine -MAINTAINER "Denver Williams " -ENV TERRAFORM_VERSION=0.9.0-beta2 -ENV KUBECTL_VERSION=v1.5.2 -ENV ARC=amd64 -ENV AWS_CONFIG_FILE=/cncf/data/awsconfig -ENV KUBECONFIG=/cncf/data/kubeconfig -# Install AWS CLI + Deps -RUN apk add --update git bash util-linux wget tar curl build-base jq openssh bind-tools && \ - rm /var/cache/apk/* - - -#Install Kubectl -RUN wget -O /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/$ARC/kubectl && \ -chmod +x /usr/local/bin/kubectl - -# Install Terraform -RUN wget https://releases.hashicorp.com/terraform/$TERRAFORM_VERSION/terraform_"${TERRAFORM_VERSION}"_linux_$ARC.zip -RUN unzip terraform*.zip -d /usr/bin - - -# Install CFSSL -RUN go get -u github.com/cloudflare/cfssl/cmd/cfssl && \ -#Add Terraform Modules -go get -u github.com/cloudflare/cfssl/cmd/... - -WORKDIR /cncf/data -COPY entrypoint.sh /cncf/ -COPY azure /azure/ -RUN chmod +x /cncf/entrypoint.sh - -ENTRYPOINT ["/cncf/entrypoint.sh"] -CMD ["deploy"] diff --git a/azure/cloud-config.tf b/azure/cloud-config.tf deleted file mode 100644 index 1946ffd..0000000 --- a/azure/cloud-config.tf +++ /dev/null @@ -1,37 +0,0 @@ -#Gen Certs -resource "null_resource" "cloud_gen" { - - provisioner "local-exec" { - command = < ${ var.data-dir }/azure-config.json -{ - "aadClientId": "$${ARM_CLIENT_ID}", - "aadClientSecret": "$${ARM_CLIENT_SECRET}", - "tenantId": "$${ARM_TENANT_ID}", - "subscriptionId": "$${ARM_SUBSCRIPTION_ID}", - "resourceGroup": "${ var.name }", - "location": "${ var.location }", - "subnetName": "${ var.name }", - "securityGroupName": "${ var.name }", - "vnetName": "${ var.name }", - "routeTableName": "${ var.name }", - "primaryAvailabilitySetName": "${ var.name }" -} -JSON -COMMAND - } - - provisioner "local-exec" { - when = "destroy" - on_failure = "continue" - command = < ./tmp/kubeconfig -${data.template_file.kubeconfig.rendered} -__USERDATA__ -LOCAL_EXEC - } - - provisioner "local-exec" { - command = <_"If you wish to make an apple pie from scratch, you must first invent the universe."_ -- [Carl Sagan](https://www.youtube.com/watch?v=7s664NsLeFM) - -Starting out on AWS one might be tempted to quick start from the web console and opt for Amazon Linux AMI. But that is not a portable choice. So at least for the sake of portability (perhaps in the future you'll want to run on another cloud provider, bare metal, or your laptop) it is better to opt for something like CentOS, Debian, or CoreOS. - -This is not an easy choice and there is no universal answer. Each option brings along its own dependencies, problems, and bugs. But since choose we must we will go down the CentOS direction of this decision tree and see how far it takes us. - -### CentOS 7 - -[Official CentOS images](https://wiki.centos.org/Cloud/AWS) are provided for us on the AWS Marketplace. - -To avoid ending up with deprecated AMI's and outdated images it is recommended to grab the AMI id programmatically (`aws --region us-west-2 ec2 describe-images --owners aws-marketplace --filters Name=product-code,Values=aw0evgkw8e5c1q413zgy5pjce`) and in case of a response with multiple ids pick the one with the most recent creation date as well as running `yum update` as the first step in your build process. - -### Default Docker is strongly discouraged for production use - -Docker is not actually a hard requirement for Kubernetes, but this isn't about recommending alternative container runtimes. This is about the defaults being a hidden minefield. - -#### The warning - -What happens with the common `yum install docker`? - -> $ docker info - - -``` -Containers: 0 -Server Version: 1.10.3 -Storage Driver: devicemapper - Pool Name: docker-202:1-9467182-pool - Pool Blocksize: 65.54 kB - Base Device Size: 10.74 GB - Backing Filesystem: xfs - Data file: /dev/loop0 - Metadata file: /dev/loop1 - .. - Data loop file: /var/lib/docker/devicemapper/devicemapper/data - WARNING: Usage of loopback devices is strongly discouraged for production use. - Metadata loop file: /var/lib/docker/devicemapper/devicemapper/metadata -``` - -As you can see from the warning, the default Docker storage config that ships with CentOS 7 is not recommended for production use. Using devicemapper with loopback can lead to unpredictable behaviour. - -In fact, to give a bit of a look into this dead end if we follow the path all the way to a Kubernetes cluster you will see nodes coming up like this: - -``` -systemctl --failed - UNIT LOAD ACTIVE SUB DESCRIPTION -● docker-storage-setup.service loaded failed failed Docker Storage Setup -● kdump.service loaded failed failed Crash recovery kernel arming -● network.service loaded failed failed LSB: Bring up/down networking -``` - -#### What docker-storage-setup is trying (and failing) to do - -`docker-storage-setup` looks for free space in the volume group of the root volume and attempts to setup a thin pool. If there is no free space it fails to set up a LVM thin pool and will fall back to using loopback devices. Which we are warned by docker itself is a `strongly discouraged` outcome. - -#### Why this is a problem - -This is insidious for several reasons. Depending on how many volumes your instance happens to spin up with (and how they're configured) you might never see this warning or experience any problem at all. For example if you have one hard-drive on bare-metal and no unallocated space this will always happen. - -If the disk provisioning changes you might end up in this edge case but the cluster will _still_ initially appear to be working. Only after some activity will xfs corruption in the docker image tree (`/var/lib/docker`) start to sporadically manifest itself and kubernetes nodes will mysteriously fail as a result. - - -Despite this being [known as problematic](https://twitter.com/codinghorror/status/604096348682485760) for some time and [documented](https://access.redhat.com/documentation/en/red-hat-enterprise-linux-atomic-host/7/single/getting-started-with-containers/#overlay_graph_driver), people still frequently [run into this](https://forums.docker.com/t/docker-with-devicemapper-doesnt-start-on-centos-7/9641) problem. - -Incidentally `yum install docker` can result in slightly different versions of docker installed. - -> Each docker release has some known issues running in Kubernetes as a runtime. - -[So what's the recommended docker version](https://github.com/kubernetes/kubernetes/issues/25893 )? v1.12 or v1.11? It turns out the latest (v1.12) is not yet supported by Kubernetes v1.4. - -_The problem is a distribution like CentOS 7, officially supported by Kubernetes, by default will arbitrarily work for some but not others, with the full requirements hidden and **underspecified**._ - -At the very least Docker versions should be pinned together with OS and Kubernetes versions and a recommendation about the storage driver should be made. - - -#### To avoid these pitfalls, carefully select the storage driver - -As Docker has a [pluggable storage driver architecture](https://docs.docker.com/engine/userguide/storagedriver/selectadriver/) and the default is (or might be) inappropriate, you must carefully consider your options. As discussed, getting this wrong will eventually cascade all the way to hard to debug and reproduce bugs and broken clusters. - ->Which storage driver should you choose? -Several factors influence the selection of a storage driver. However, these two facts must be kept in mind: - -> - No single driver is well suited to every use-case -> - Storage drivers are improving and evolving all of the time - -The docker docs don't take a position either. If one doesn't want to make assumptions about how many disks a machine has (laptops, bare metal servers with one drive, 'etc) direct LVM is out. - -AUFS [was the original backend](http://jpetazzo.github.io/assets/2015-03-03-not-so-deep-dive-into-docker-storage-drivers.html#28) used by docker but is not in the mainline kernel (it is however included by debian/ubuntu). - -Overlay is in mainline and supported as a Technology Preview by RHEL. - -Additionally _"Many people consider OverlayFS as the future of the Docker storage driver"_. It is the future proof way to go. - -##### Overlay Dependencies - -- CentOS 7.2 -- _"Only XFS is currently supported for use as a lower layer file system."_ -- _"/etc/sysconfig/docker must not contain --selinux-enabled"_ (for now) - -With the above satisfied, to enable overlay simply: - -`echo "overlay" > /etc/modules-load.d/overlay.conf` - -And add the flag (`--storage-driver=overlay`) in the docker service file or DOCKER_OPTS (`/etc/default/docker`). - -This requires a reboot, but first... - -### Properly configure netfilter - -`docker info` had another complaint. - -``` -WARNING: bridge-nf-call-iptables is disabled -WARNING: bridge-nf-call-ip6tables is disabled -``` - -This toggles whether packets traversing the bridge are forwarded to iptables. -This is [docker issue #24809](https://github.com/docker/docker/issues/24809) and _could_ be ignored ("either /proc/sys/net/bridge/bridge-nf-call-iptables doesn't exist or is set to 0"). CentOS and most distros default this to 0. - -If I was writing a choose your own adventure book this is the point I'd write that thunder rumbles in the distance, a quiet intensity. - -If you follow _this_ dead end all the way to a Kubernetes cluster you will find out that **kube-proxy requires that bridged traffic passes through netfilter**. So that path should absolutely exist otherwise you have a problem. - -Furthermore you'll find that [kube-proxy will not work properly with Weave on Centos](https://github.com/kubernetes/kubernetes/issues/33790) if this isn't toggled to 1. At first everything will appear to be fine, the problem only manifests itself by kubernetes service endpoints not being routable. - -To get rid of these warnings you might try: - -`echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables` - -`echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables` - -This would toggle the setting but not persist after a reboot. Once again, this will cause a situation where a cluster will initially appear to work fine. - -The above settings used to live in /etc/sysctl.conf the contents of which nowadays are: - -``` -# System default settings live in /usr/lib/sysctl.d/00-system.conf. -# To override those settings, enter new settings here, or in an /etc/sysctl.d/.conf file -``` - -This file is sourced on every invocation of `sysctl -p`. - -Attempting to toggle via `sysctl -p` [gives the following error](https://github.com/ansible/ansible/issues/6272) under certain conditions: - -``` -error: "net.bridge.bridge-nf-call-ip6tables" is an unknown key -error: "net.bridge.bridge-nf-call-iptables" is an unknown key -``` - -Since `sysctl` runs at boot there's also a very possible race condition [if the bridge module hasn't loaded yet](https://bugzilla.redhat.com/show_bug.cgi?id=1054178#c1) at that point. Making this a (sometimes) misleading error message. - -The correct way to set this as of CentOS7: - -> $ cat /usr/lib/sysctl.d/00-system.conf - -``` - -# Kernel sysctl configuration file -# -# For binary values, 0 is disabled, 1 is enabled. See sysctl(8) and -# sysctl.conf(5) for more details. - -# Disable netfilter on bridges. -net.bridge.bridge-nf-call-ip6tables = 0 -net.bridge.bridge-nf-call-iptables = 0 -net.bridge.bridge-nf-call-arptables = 0 - -``` - -> $ cat /usr/lib/sysctl.d/90-system.conf - -``` -net.bridge.bridge-nf-call-ip6tables = 1 -net.bridge.bridge-nf-call-iptables = 1 -``` - -This way systemd ensures these settings will be evaluated whenever a bridge module is loaded and the race condition is avoided. - -Speaking of misleading error messages, kubernetes logs an [incorrect br-netfilter warning on Centos 7](https://github.com/kubernetes/kubernetes/issues/23385): - -> proxier.go:205] missing br-netfilter module or unset br-nf-call-iptables; proxy may not work as intended - -Stay the course, there's nothing else to toggle to make this warning go away, it is simply a false positive. - -### Consider disabling selinux - -With Overlay as the storage backend currently you can only run with selinux on the host, a temporary limitation. - -However, elsewhere, kubernetes uses a mechanism that injects special volumes into each container to expose service account tokens and [with selinux turned on secrets simply don't work](http://stackoverflow.com/questions/35338213/kubernetes-serviceaccounts-and-selinux/35347520#35347520). - -The work around is to set the security context of volume on the kubernetes host (`sudo chcon -Rt svirt_sandbox_file_t /var/lib/kubelet`) or set selinux to permissive mode. - -Otherwise down the line [kubernetes add-ons](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons) will fail or behave unpredictably. For example KubeDNS will fail to authenticate with the master and [dns lookups on service endpoints will fail](https://github.com/cncf/demo/issues/103). (Slightly differs from the bridge netfilter disabled problem described above which results in routing by ip intermittently failing) - -Since there might be other selinux permissions necessary elsewhere consider turning off selinux entirely until this is properly decided upon and documented. - -### Correct CNI config - - -Kubernetes supports [CNI Network Plugins](http://kubernetes.io/docs/admin/network-plugins/#cni) for interoperability. Setting up a network overlay requires this dependency. - -Kubernetes 1.3.5 [broke the cni config](https://github.com/kubernetes/kubernetes/issues/30681) — as of that version it is necessary to pull in the [cni release binaries](https://github.com/containernetworking/cni/releases) into the cni bin folder. - -As of Kuberentes 1.4 the [flags to specify cni directories](https://github.com/kubernetes/kubernetes.github.io/pull/1516) changed and documentation was added pinning the minimum cni version to 0.2 and at least the `lo` binary. - -### Other Dependencies - -There's [additional undocumented missing dependencies](https://github.com/cncf/demo/issues/64) as follows: - - - conntrack-tools - - socat - - bridge-utils - -### AWS specific requirements & debugging - -[Peeking under the hood of Kubernetes on AWS](https://github.com/kubernetes/kubernetes/blob/master/docs/design/aws_under_the_hood.md#tagging) you'll find: - -> All AWS resources are tagged with a tag named "KubernetesCluster", with a value that is the unique cluster-id. This tag is used to identify a particular 'instance' of Kubernetes, even if two clusters are deployed into the same VPC. Resources are considered to belong to the same cluster if and only if they have the same value in the tag named "KubernetesCluster". - -This isn't only necessary to differentiate resources between different clusters in the same VPC but also needed for the controller to discover and manage AWS resources at all (even if it has an entire VPC to itself). - -Unfortunately these tags are [not filtered on in a uniform manner across different resource types](https://github.com/cncf/demo/issues/144). - -A `kubectl create -f resource.yaml` successfully submitted to kubernetes might not result in expected functionality (in this case a load balancer endpoint) even when the desired resource shows as `creating...`. It will simply show that indefinitely instead of an error. - -Since the problem doesn't bubble up to kubectl responses the only way to see that something is amiss is by carefully watching the controller log. - -``` -aws.go:2731] Error opening ingress rules for the load balancer to the instances: Multiple tagged security groups found for instance i-04bd9c4c8aa; ensure only the k8s security group is tagged -``` - -[Reading the code](https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/providers/aws/aws.go#L2783) yields: - -``` -// Returns the first security group for an instance, or nil -// We only create instances with one security group, so we don't expect multiple security groups. -// However, if there are multiple security groups, we will choose the one tagged with our cluster filter. -// Otherwise we will return an error. -``` - -In this example the kubernetes masters and minions each have a security group, both security groups are tagged with "KubernetesCluster=name". Removing the tags from the master security group resolves this problem as now the controller receives an expected response from the AWS API. It is easy to imagine many other scenarios where such conflicts might arise if the tag filtering is not consistent. - -Smoke tests that simply launch and destroy a large amount of pods and resources would not catch this problem either. - -## Conclusion - -The most difficult bugs are the ones that occur far away from their origins. -Bugs that will slowly but surely degrade a cluster and yet sneak past continuous integration tests. - -Additionally the target is a moving one. Minor releases of kubernetes can still have undocumented changes and undocumented dependencies. - -If a [critical Add-Ons fails](https://github.com/kubernetes/kubernetes/issues/14232) seemingly identical clusters deployed minutes apart will have divergent behaviour. The cloud environments clusters slot into are also a source of state and therfore subtle edgecase that can confuse the controller and silently prevent it from deploying things. - -In short, this is a complex support matrix. - -A possible way to improve things is by introducing: - -- A set of host OS images with minimal changes baked in as neccessary for Kubernetes - - - Continuously (weekly?) rebased on top of the latest official images - - - - As the basis for a well documented reference implementation of a custom cluster - -- Long running custom clusters spun up for each permutation of minor version updates (kubernetes version bump, weave, flannel, etcd, and so on) - -- A deterministic demo app/deployment as a comprehensive smoketest & benchmark - - -The community need to mix and match the multiple supported components with arbitrary neccessary for custom deployments can be benefit from a set of "blessed" kubernetes-flavored host OS images and a more typical real-world artifcat to check their customizations against. \ No newline at end of file diff --git a/docs/k8s-cube.png b/docs/k8s-cube.png deleted file mode 100644 index a483e64..0000000 Binary files a/docs/k8s-cube.png and /dev/null differ diff --git a/docs/k8s-simpler.png b/docs/k8s-simpler.png deleted file mode 100644 index e3423c4..0000000 Binary files a/docs/k8s-simpler.png and /dev/null differ diff --git a/docs/sdn.png b/docs/sdn.png deleted file mode 100644 index 9c03484..0000000 Binary files a/docs/sdn.png and /dev/null differ diff --git a/docs/src/arch.graffle b/docs/src/arch.graffle deleted file mode 100644 index d74ec97..0000000 Binary files a/docs/src/arch.graffle and /dev/null differ diff --git a/docs/src/k8s-simpler.graffle b/docs/src/k8s-simpler.graffle deleted file mode 100644 index c5041cd..0000000 Binary files a/docs/src/k8s-simpler.graffle and /dev/null differ diff --git a/docs/src/sdn.graffle b/docs/src/sdn.graffle deleted file mode 100644 index 3d8d29d..0000000 Binary files a/docs/src/sdn.graffle and /dev/null differ diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100644 index cdcb4da..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# -# RUN ENTRYPOINT. - -set -e -RED='\033[0;31m' -NC='\033[0m' # No Color - -# Run CMD -export TF_VAR_name="$2" -if [ "$1" = "deploy" ] ; then - terraform get /azure && \ - terraform apply -target null_resource.sshkey_gen /azure && \ - terraform apply -target null_resource.ssl_gen /azure && \ - terraform apply -target null_resource.cloud_gen /azure && \ - terraform apply -target module.dns.null_resource.dns_gen /azure && \ - terraform apply -target module.etcd.azurerm_network_interface.cncf /azure && \ - time terraform apply /azure && printf "${RED}\n#Commands to Configue Kubectl \n\n" && printf 'sudo chown -R $(whoami):$(whoami) $(pwd)/data/${name} \n\n' && printf 'export KUBECONFIG=$(pwd)/data/${name}/kubeconfig \n\n'${NC} -elif [ "$1" = "destroy" ] ; then - time terraform destroy -force /azure -fi - diff --git a/provisioning/Dockerfile b/provisioning/Dockerfile new file mode 100644 index 0000000..94fae67 --- /dev/null +++ b/provisioning/Dockerfile @@ -0,0 +1,55 @@ +FROM golang:alpine +MAINTAINER "Denver Williams " +ENV KUBECTL_VERSION=v1.5.2 +ENV GCLOUD_VERSION=150.0.0 +ENV AWSCLI_VERSION=1.11.75 +ENV AZURECLI_VERSION=2.0.2 +ENV PACKETCLI_VERSION=1.33 +ENV TERRAFORM_VERSION=0.9.4 +ENV ARC=amd64 + +# Install AWS / AZURE CLI Deps +RUN apk update +RUN apk add --update git bash util-linux wget tar curl build-base jq \ + py-pip groff less openssh bind-tools python python-dev libffi-dev openssl-dev + +# no way to pin this packet-cli at the moment +RUN go get -u github.com/ebsarr/packet +RUN pip install packet-python==${PACKETCLI_VERSION} argh tabulate +RUN pip install azure-cli==${AZURECLI_VERSION} +RUN pip install awscli==${AWSCLI_VERSION} + +RUN apk --purge -v del py-pip && \ + rm /var/cache/apk/* + +# Install Google Cloud SDK +RUN wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_VERSION}-linux-x86.tar.gz && \ +tar xvfz google-cloud-sdk-${GCLOUD_VERSION}-linux-x86.tar.gz && \ +./google-cloud-sdk/install.sh -q + + +#Install Kubectl +RUN wget -O /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/$ARC/kubectl && \ +chmod +x /usr/local/bin/kubectl + +# Install Terraform +RUN wget https://releases.hashicorp.com/terraform/$TERRAFORM_VERSION/terraform_"${TERRAFORM_VERSION}"_linux_$ARC.zip +RUN unzip terraform*.zip -d /usr/bin + +# Install CFSSL +RUN go get -u github.com/cloudflare/cfssl/cmd/cfssl && \ +go get -u github.com/cloudflare/cfssl/cmd/... + +# Install Gzip+base64 Provider +RUN go get -u github.com/jakexks/terraform-provider-gzip && \ + echo providers { >> ~/.terraformrc && \ + echo ' gzip = "terraform-provider-gzip"' >> ~/.terraformrc && \ + echo } >> ~/.terraformrc + +#Add Terraform Modules + +COPY provision.sh /cncf/ +RUN chmod +x /cncf/provision.sh +ENTRYPOINT ["/cncf/provision.sh"] +WORKDIR /cncf/ +CMD ["aws-deploy"] diff --git a/provisioning/aws/Readme.mkd b/provisioning/aws/Readme.mkd new file mode 100644 index 0000000..810b7c0 --- /dev/null +++ b/provisioning/aws/Readme.mkd @@ -0,0 +1,79 @@ +## Prerequisites +* [docker](https://docker.io/) + +* AWS User with following Permissions: + - AmazonEC2FullAccess + - AmazonS3FullAccess + - AWSCodeDeployFullAccess + - AmazonRoute53DomainsFullAccess + - AmazonRoute53FullAccess + - IAMFullAccess + - IAMUserChangePassword + +* Must use a config config from the repo data/terraform.tfvars + +## export AWS Authentication + +``` +export AWS_ACCESS_KEY_ID="YOUR_AWS_KEY_ID" +export AWS_SECRET_ACCESS_KEY="YOUR_AWS_SECRET_KEY" +``` + +## create AWS Kubernetes Endpoint + +``` +docker run -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -v $(pwd)/data:/cncf/data create/aws +``` + +## configure kubectl on local system + +``` +sudo chown -R $(whoami):$(whoami) data/ +export KUBECONFIG=$(pwd)/data/kubeconfig +$ kubectl get nodes +NAME STATUS AGE +ip-10-0-10-10.ap-southeast-2.compute.internal Ready,SchedulingDisabled 7m +ip-10-0-10-11.ap-southeast-2.compute.internal Ready,SchedulingDisabled 6m +ip-10-0-10-12.ap-southeast-2.compute.internal Ready,SchedulingDisabled 7m +ip-10-0-10-51.ap-southeast-2.compute.internal Ready 6m +ip-10-0-11-7.ap-southeast-2.compute.internal Ready 6m +ip-10-0-12-68.ap-southeast-2.compute.internal Ready 6m +``` + +## data folder contains certs + kubeconfig + +It also contains a json file containing details on current cluster state. + +``` +$ sudo cat ./data/kubeconfig +apiVersion: v1 +clusters: +- cluster: + certificate-authority: .cfssl/ca.pem + server: https://kz8s-apiserver-test-453655923.ap-southeast-2.elb.amazonaws.com + name: cluster-test +contexts: +- context: + cluster: cluster-test + user: admin-test + name: test +current-context: test +kind: Config +preferences: {} +users: +- name: admin-test + user: + client-certificate: .cfssl/k8s-admin.pem + client-key: .cfssl/k8s-admin-key.pem +``` + + +## destroy AWS Kubernetes Endpoint + +``` +docker run -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -v $(pwd)/data:/cncf/data terminate/aws +``` diff --git a/provisioning/aws/aws.tf b/provisioning/aws/aws.tf new file mode 100644 index 0000000..89afffa --- /dev/null +++ b/provisioning/aws/aws.tf @@ -0,0 +1,8 @@ +provider "aws" { } +provider "gzip" {compressionlevel = "BestCompression"} + +# configured via: +# $ export AWS_ACCESS_KEY_ID="anaccesskey" +# $ export AWS_SECRET_ACCESS_KEY="asecretkey" +# $ export AWS_DEFAULT_REGION="us-west-2" +# https://www.terraform.io/docs/providers/aws/#environment-variables diff --git a/azure/cert.tf b/provisioning/aws/cert.tf similarity index 66% rename from azure/cert.tf rename to provisioning/aws/cert.tf index 49eeb10..0c3dd3a 100644 --- a/azure/cert.tf +++ b/provisioning/aws/cert.tf @@ -1,13 +1,13 @@ -#Gen Certs +# Gen Certs resource "null_resource" "ssl_gen" { provisioner "local-exec" { command = <ca-csr.json +echo "$(ca-config)" >ca-config.json + +# generate ca +cfssl gencert -initca ca-csr.json | cfssljson -bare ca - +_chmod ca + +# generate keys and certs +generate k8s-admin client-server "${DEFAULT_HOSTS},*.*.compute.internal,*.ec2.internal" +generate k8s-apiserver client-server "${DEFAULT_HOSTS},${K8S_SERVICE_IP},master.${INTERNAL_TLD},*.${AWS_REGION}.elb.amazonaws.com" +generate k8s-etcd client-server "etcd.${INTERNAL_TLD},etcd1.${INTERNAL_TLD},etcd2.${INTERNAL_TLD},etcd3.${INTERNAL_TLD}" +generate k8s-worker client "${DEFAULT_HOSTS},*.*.compute.internal,*.ec2.internal" + +# TODO: fix cert provisioning hacks +# tar -rf k8s-apiserver.tar k8s-etcd.pem k8s-etcd-key.pem +# tar -rf k8s-worker.tar ca.pem +# bzip2 k8s-apiserver.tar +# bzip2 k8s-worker.tar diff --git a/provisioning/aws/input.tf b/provisioning/aws/input.tf new file mode 100644 index 0000000..0983dd9 --- /dev/null +++ b/provisioning/aws/input.tf @@ -0,0 +1,36 @@ +variable "name" { default = "aws" } + +variable "internal_tld" { default = "aws.cncf.demo" } +variable "data_dir" { default = "/cncf/data/aws" } + +# AWS Cloud Specific Settings +variable "aws_region" { default = "ap-southeast-2" } +variable "aws_key_name" { default = "aws" } +variable "aws_azs" { default = "ap-southeast-2a,ap-southeast-2b,ap-southeast-2c" } +variable "vpc_cidr" { default = "10.0.0.0/16" } +variable "allow_ssh_cidr" { default = "0.0.0.0/0" } + +# VM Image and size +variable "admin_username" { default = "core" } +variable "aws_image_ami" { default = "ami-fde3e09e"} # channel/stable type/hvm +variable "aws_master_vm_size" { default = "m3.medium" } +variable "aws_worker_vm_size" { default = "m3.medium" } +variable "aws_bastion_vm_size" { default = "t2.nano" } + +# Kubernetes +variable "cluster_domain" { default = "cluster.local" } +variable "pod_cidr" { default = "10.2.0.0/16" } +variable "service_cidr" { default = "10.3.0.0/24" } +variable "k8s_service_ip" { default = "10.3.0.1" } +variable "dns_service_ip" { default = "10.3.0.10" } +variable "master_node_count" { default = "3" } +variable "worker_node_count" { default = "3" } +variable "worker_node_min" { default = "3" } +variable "worker_node_max" { default = "5" } + +# Deployment Artifact Versions +# Hyperkube +# Set from https://quay.io/repository/coreos/hyperkube?tab=tags +variable "kubelet_image_url" { default = "quay.io/coreos/hyperkube"} +variable "kubelet_image_tag" { default = "v1.5.1_coreos.0"} + diff --git a/provisioning/aws/keypair.tf b/provisioning/aws/keypair.tf new file mode 100644 index 0000000..70a3c25 --- /dev/null +++ b/provisioning/aws/keypair.tf @@ -0,0 +1,17 @@ +# Add AWS Keypair +resource "null_resource" "aws_keypair" { + provisioner "local-exec" { + command = < ${ var.data_dir }/${ var.aws_key_name }.pem +chmod 400 ${ var.data_dir }/${ var.aws_key_name }.pem +EOF + } +} + +resource "null_resource" "dummy_dependency2" { + depends_on = [ "null_resource.aws_keypair" ] +} diff --git a/provisioning/aws/modules.tf b/provisioning/aws/modules.tf new file mode 100644 index 0000000..d39d214 --- /dev/null +++ b/provisioning/aws/modules.tf @@ -0,0 +1,120 @@ +module "vpc" { + source = "./modules/vpc" + name = "${ var.name }" + + azs = "${ var.aws_azs }" + cidr = "${ var.vpc_cidr }" +} + +module "security" { + source = "./modules/security" + name = "${ var.name }" + + vpc_cidr = "${ var.vpc_cidr }" + vpc_id = "${ module.vpc.id }" + allow_ssh_cidr = "${ var.allow_ssh_cidr }" +} + + +module "iam" { + source = "./modules/iam" + name = "${ var.name }" +} + + +module "dns" { + source = "./modules/dns" + name = "${ var.name }" + master_node_count = "${ var.master_node_count }" + master_ips = "${ module.etcd.master_ips }" + internal_tld = "${ var.internal_tld }" + vpc_id = "${ module.vpc.id }" +} + +module "etcd" { + source = "./modules/etcd" + depends_id = "${ module.dns.depends_id }" + instance_profile_name = "${ module.iam.instance_profile_name_master }" + + master_node_count = "${ var.master_node_count }" + name = "${ var.name }" + ami_id = "${ var.aws_image_ami }" + key_name = "${ var.aws_key_name }" + cluster_domain = "${ var.cluster_domain }" + kubelet_image_url = "${ var.kubelet_image_url }" + kubelet_image_tag = "${ var.kubelet_image_tag }" + dns_service_ip = "${ var.dns_service_ip }" + etcd_security_group_id = "${ module.security.etcd_id }" + external_elb_security_group_id = "${ module.security.external_elb_id }" + instance_type = "${ var.aws_master_vm_size }" + internal_tld = "${ var.internal_tld }" + pod_cidr = "${ var.pod_cidr }" + region = "${ var.aws_region }" + service_cidr = "${ var.service_cidr }" + subnet_ids_private = "${ module.vpc.subnet_ids_private }" + subnet_ids_public = "${ module.vpc.subnet_ids_public }" + vpc_id = "${ module.vpc.id }" + ca = "${file("${ var.data_dir }/.cfssl/ca.pem")}" + k8s_etcd = "${file("${ var.data_dir }/.cfssl/k8s-etcd.pem")}" + k8s_etcd_key = "${file("${ var.data_dir }/.cfssl/k8s-etcd-key.pem")}" + k8s_apiserver = "${file("${ var.data_dir }/.cfssl/k8s-apiserver.pem")}" + k8s_apiserver_key = "${file("${ var.data_dir }/.cfssl/k8s-apiserver-key.pem")}" +} + +module "bastion" { + source = "./modules/bastion" + + ami_id = "${ var.aws_image_ami }" + instance_type = "${ var.aws_bastion_vm_size }" + internal_tld = "${ var.internal_tld }" + key_name = "${ var.aws_key_name }" + name = "${ var.name }" + security_group_id = "${ module.security.bastion_id }" + subnet_ids = "${ module.vpc.subnet_ids_public }" + vpc_id = "${ module.vpc.id }" +} + +module "worker" { + source = "./modules/worker" + depends_id = "${ module.dns.depends_id }" + instance_profile_name = "${ module.iam.instance_profile_name_worker }" + + ami_id = "${ var.aws_image_ami }" + capacity = { + desired = "${ var.worker_node_count}" + max = "${ var.worker_node_max}" + min = "${ var.worker_node_min}" + } + cluster_domain = "${ var.cluster_domain }" + kubelet_image_url = "${ var.kubelet_image_url }" + kubelet_image_tag = "${ var.kubelet_image_tag }" + dns_service_ip = "${ var.dns_service_ip }" + instance_type = "${ var.aws_worker_vm_size }" + internal_tld = "${ var.internal_tld }" + key_name = "${ var.aws_key_name }" + name = "${ var.name }" + region = "${ var.aws_region }" + security_group_id = "${ module.security.worker_id }" + subnet_ids = "${ module.vpc.subnet_ids_private }" + ca = "${file("${ var.data_dir }/.cfssl/ca.pem")}" + k8s_worker = "${file("${ var.data_dir }/.cfssl/k8s-worker.pem")}" + k8s_worker_key = "${file("${ var.data_dir }/.cfssl/k8s-worker-key.pem")}" + + volume_size = { + ebs = 250 + root = 52 + } + vpc_id = "${ module.vpc.id }" + worker_name = "general" +} + +module "kubeconfig" { + source = "../kubeconfig" + + admin_key_pem = "${ var.data_dir }/.cfssl/k8s-admin-key.pem" + admin_pem = "${ var.data_dir }/.cfssl/k8s-admin.pem" + ca_pem = "${ var.data_dir }/.cfssl/ca.pem" + data_dir = "${ var.data_dir }" + fqdn_k8s = "${ module.etcd.external_elb }" + name = "${ var.name }" +} diff --git a/provisioning/aws/modules/bastion/ec2.tf b/provisioning/aws/modules/bastion/ec2.tf new file mode 100644 index 0000000..a1697fc --- /dev/null +++ b/provisioning/aws/modules/bastion/ec2.tf @@ -0,0 +1,36 @@ +resource "aws_instance" "bastion" { + ami = "${ var.ami_id }" + associate_public_ip_address = true + instance_type = "${ var.instance_type }" + key_name = "${ var.key_name }" + + # TODO: force private_ip to prevent collision with etcd machines + + source_dest_check = false + subnet_id = "${ element( split(",", var.subnet_ids), 0 ) }" + + tags { + builtWith = "terraform" + kz8s = "${ var.name }" + Name = "kz8s-bastion" + role = "bastion" + } + + user_data = "${ data.template_file.user-data.rendered }" + + vpc_security_group_ids = [ + "${ var.security_group_id }", + ] +} + +data "template_file" "user-data" { + template = "${ file( "${ path.module }/user-data.yml" )}" + + vars { + internal_tld = "${ var.internal_tld }" + } +} + +resource "null_resource" "dummy_dependency" { + depends_on = [ "aws_instance.bastion" ] +} diff --git a/provisioning/aws/modules/bastion/input.tf b/provisioning/aws/modules/bastion/input.tf new file mode 100644 index 0000000..d06cfe6 --- /dev/null +++ b/provisioning/aws/modules/bastion/input.tf @@ -0,0 +1,8 @@ +variable "ami_id" {} +variable "instance_type" {} +variable "internal_tld" {} +variable "key_name" {} +variable "name" {} +variable "security_group_id" {} +variable "subnet_ids" {} +variable "vpc_id" {} diff --git a/provisioning/aws/modules/bastion/output.tf b/provisioning/aws/modules/bastion/output.tf new file mode 100644 index 0000000..94de6c7 --- /dev/null +++ b/provisioning/aws/modules/bastion/output.tf @@ -0,0 +1,2 @@ +output "depends_id" { value = "${null_resource.dummy_dependency.id}" } +output "ip" { value = "${ aws_instance.bastion.public_ip }" } diff --git a/provisioning/aws/modules/bastion/user-data.yml b/provisioning/aws/modules/bastion/user-data.yml new file mode 100644 index 0000000..41ece8a --- /dev/null +++ b/provisioning/aws/modules/bastion/user-data.yml @@ -0,0 +1,14 @@ +#cloud-config + +--- +coreos: + update: + reboot-strategy: etcd-lock + + etcd2: + discovery-srv: ${ internal_tld } + proxy: on + + units: + - name: etcd2.service + command: start diff --git a/provisioning/aws/modules/dns/dns.tf b/provisioning/aws/modules/dns/dns.tf new file mode 100644 index 0000000..eeb3d8d --- /dev/null +++ b/provisioning/aws/modules/dns/dns.tf @@ -0,0 +1,58 @@ +resource "aws_route53_zone" "internal" { + comment = "Kubernetes cluster DNS (internal)" + name = "${ var.internal_tld }" + tags { + builtWith = "terraform" + KubernetesCluster = "${ var.name }" + Name = "k8s-${ var.name }" + } + vpc_id = "${ var.vpc_id }" +} + +resource "aws_route53_record" "A-etcd" { + name = "etcd" + records = [ "${ var.master_ips }" ] + ttl = "300" + type = "A" + zone_id = "${ aws_route53_zone.internal.zone_id }" +} + +resource "aws_route53_record" "A-etcds" { + name = "etcd${ count.index+1 }" + count = "${ var.master_node_count }" + ttl = "300" + type = "A" + records = [ "${ element(var.master_ips, count.index) }" ] + zone_id = "${ aws_route53_zone.internal.zone_id }" +} + +resource "aws_route53_record" "CNAME-master" { + name = "master" + records = [ "etcd.${ var.internal_tld }" ] + ttl = "300" + type = "CNAME" + zone_id = "${ aws_route53_zone.internal.zone_id }" +} + +resource "aws_route53_record" "etcd-client-tcp" { + name = "_etcd-client._tcp" + ttl = "300" + type = "SRV" + records = [ "${ formatlist("0 0 2379 %v", aws_route53_record.A-etcds.*.fqdn) }" ] + zone_id = "${ aws_route53_zone.internal.zone_id }" +} + +resource "aws_route53_record" "etcd-server-tcp" { + name = "_etcd-server-ssl._tcp" + ttl = "300" + type = "SRV" + records = [ "${ formatlist("0 0 2380 %v", aws_route53_record.A-etcds.*.fqdn) }" ] + zone_id = "${ aws_route53_zone.internal.zone_id }" +} + +resource "null_resource" "dummy_dependency" { + depends_on = [ + "aws_route53_record.etcd-server-tcp", + "aws_route53_record.A-etcd", + ] +} diff --git a/provisioning/aws/modules/dns/input.tf b/provisioning/aws/modules/dns/input.tf new file mode 100644 index 0000000..4c2eeca --- /dev/null +++ b/provisioning/aws/modules/dns/input.tf @@ -0,0 +1,6 @@ +variable "internal_tld" {} +variable "name" {} +variable "vpc_id" {} +variable "master_ips" { type = "list"} +variable "master_node_count" {} + diff --git a/provisioning/aws/modules/dns/output.tf b/provisioning/aws/modules/dns/output.tf new file mode 100644 index 0000000..2c05297 --- /dev/null +++ b/provisioning/aws/modules/dns/output.tf @@ -0,0 +1,3 @@ +output "depends_id" { value = "${null_resource.dummy_dependency.id}" } +output "internal_name_servers" { value = "${ aws_route53_zone.internal.name_servers }" } +output "internal_zone_id" { value = "${ aws_route53_zone.internal.zone_id }" } diff --git a/provisioning/aws/modules/etcd/cloud-config.tf b/provisioning/aws/modules/etcd/cloud-config.tf new file mode 100644 index 0000000..9aed236 --- /dev/null +++ b/provisioning/aws/modules/etcd/cloud-config.tf @@ -0,0 +1,69 @@ +resource "gzip_me" "ca" { + input = "${ var.ca }" +} + +resource "gzip_me" "k8s_etcd" { + input = "${ var.k8s_etcd }" +} + +resource "gzip_me" "k8s_etcd_key" { + input = "${ var.k8s_etcd_key }" +} + +resource "gzip_me" "k8s_apiserver" { + input = "${ var.k8s_apiserver }" +} + +resource "gzip_me" "k8s_apiserver_key" { + input = "${ var.k8s_apiserver_key }" +} + +data "template_file" "kube-apiserver" { + template = "${ file( "${ path.module }/kube-apiserver.yml" )}" + + vars { + internal_tld = "${ var.internal_tld }" + service_cidr = "${ var.service_cidr }" + hyperkube = "${ var.kubelet_image_url }:${ var.kubelet_image_tag }" + kubelet_image_url = "${ var.kubelet_image_url }" + kubelet_image_tag = "${ var.kubelet_image_tag }" + } +} + +resource "gzip_me" "kube-apiserver" { + input = "${ data.template_file.kube-apiserver.rendered }" +} + +data "template_file" "cloud-config" { + count = "${ var.master_node_count }" + template = "${ file( "${ path.module }/cloud-config.yml" )}" + + vars { + cluster_domain = "${ var.cluster_domain }" + cluster-token = "etcd-cluster-${ var.name }" + dns_service_ip = "${ var.dns_service_ip }" + fqdn = "etcd${ count.index + 1 }.${ var.internal_tld }" + hostname = "etcd${ count.index + 1 }" + hyperkube = "${ var.kubelet_image_url }:${ var.kubelet_image_tag }" + kubelet_image_url = "${ var.kubelet_image_url }" + kubelet_image_tag = "${ var.kubelet_image_tag }" + internal_tld = "${ var.internal_tld }" + pod_cidr = "${ var.pod_cidr }" + region = "${ var.region }" + service_cidr = "${ var.service_cidr }" + ca = "${ gzip_me.ca.output }" + k8s_etcd = "${ gzip_me.k8s_etcd.output }" + k8s_etcd_key = "${ gzip_me.k8s_etcd_key.output }" + k8s_apiserver = "${ gzip_me.k8s_apiserver.output }" + k8s_apiserver_key = "${ gzip_me.k8s_apiserver_key.output }" + kube-apiserver-yml = "${ gzip_me.kube-apiserver.output }" + } +} + + + +# data "template_file" "kube-controller-manager" + +# data "template_file" "kube-proxy" + +# data "template_file" "kube-scheduler" diff --git a/provisioning/aws/modules/etcd/cloud-config.yml b/provisioning/aws/modules/etcd/cloud-config.yml new file mode 100644 index 0000000..27524c2 --- /dev/null +++ b/provisioning/aws/modules/etcd/cloud-config.yml @@ -0,0 +1,248 @@ +#cloud-config + +--- +coreos: + + etcd2: + advertise-client-urls: http://${ fqdn }:2379 + # cert-file: /etc/kubernetes/ssl/k8s-etcd.pem + # debug: true + discovery-srv: ${ internal_tld } + initial-advertise-peer-urls: https://${ fqdn }:2380 + initial-cluster-state: new + initial-cluster-token: ${ cluster-token } + # key-file: /etc/kubernetes/ssl/k8s-etcd-key.pem + listen-client-urls: http://0.0.0.0:2379 + listen-peer-urls: https://0.0.0.0:2380 + name: ${ hostname } + peer-trusted-ca-file: /etc/kubernetes/ssl/ca.pem + peer-client-cert-auth: true + peer-cert-file: /etc/kubernetes/ssl/k8s-etcd.pem + peer-key-file: /etc/kubernetes/ssl/k8s-etcd-key.pem + + units: + - name: etcd2.service + command: start + + - name: flanneld.service + command: start + drop-ins: + - name: 50-network-config.conf + content: | + [Service] + ExecStartPre=-/usr/bin/etcdctl mk /coreos.com/network/config \ + '{ "Network": "${ pod_cidr }", "Backend": { "Type": "vxlan" } }' + Restart=always + RestartSec=10 + + - name: docker.service + command: start + drop-ins: + - name: 40-flannel.conf + content: | + [Unit] + After=flanneld.service + Requires=flanneld.service + [Service] + Restart=always + RestartSec=10 + - name: overlay.conf + content: | + [Service] + Environment="DOCKER_OPTS=--storage-driver=overlay" + + - name: kubelet.service + command: start + content: | + [Unit] + ConditionFileIsExecutable=/usr/lib/coreos/kubelet-wrapper + [Service] + Environment="KUBELET_IMAGE_URL=${ kubelet_image_url }" + Environment="KUBELET_IMAGE_TAG=${ kubelet_image_tag }" + Environment="RKT_OPTS=\ + --volume dns,kind=host,source=/etc/resolv.conf \ + --mount volume=dns,target=/etc/resolv.conf \ + --volume rkt,kind=host,source=/opt/bin/host-rkt \ + --mount volume=rkt,target=/usr/bin/rkt \ + --volume var-lib-rkt,kind=host,source=/var/lib/rkt \ + --mount volume=var-lib-rkt,target=/var/lib/rkt \ + --volume stage,kind=host,source=/tmp \ + --mount volume=stage,target=/tmp \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log" + ExecStartPre=/usr/bin/mkdir -p /var/log/containers + ExecStartPre=/usr/bin/mkdir -p /var/lib/kubelet + ExecStartPre=/usr/bin/mount --bind /var/lib/kubelet /var/lib/kubelet + ExecStartPre=/usr/bin/mount --make-shared /var/lib/kubelet + ExecStart=/usr/lib/coreos/kubelet-wrapper \ + --allow-privileged=true \ + --api-servers=http://127.0.0.1:8080 \ + --cloud-provider=aws \ + --cluster-dns=${ dns_service_ip } \ + --cluster-domain=${ cluster_domain } \ + --config=/etc/kubernetes/manifests \ + --register-schedulable=false + Restart=always + RestartSec=5 + [Install] + WantedBy=multi-user.target + + update: + reboot-strategy: etcd-lock + +write-files: + - path: /opt/bin/host-rkt + permissions: 0755 + owner: root:root + content: | + #!/bin/sh + exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "$@" + + - path: /etc/kubernetes/manifests/kube-apiserver.yml + encoding: "gzip+base64" + content: | + ${ kube-apiserver-yml } + + - path: /etc/kubernetes/manifests/kube-controller-manager.yml + content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-controller-manager + namespace: kube-system + spec: + hostNetwork: true + containers: + - name: kube-controller-manager + image: ${ hyperkube } + command: + - /hyperkube + - controller-manager + - --cloud-provider=aws + - --leader-elect=true + - --master=http://127.0.0.1:8080 + - --root-ca-file=/etc/kubernetes/ssl/ca.pem + - --service-account-private-key-file=/etc/kubernetes/ssl/k8s-apiserver-key.pem + resources: + requests: + cpu: 200m + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10252 + initialDelaySeconds: 15 + timeoutSeconds: 1 + volumeMounts: + - mountPath: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + readOnly: true + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-host + + - path: /etc/kubernetes/manifests/kube-proxy.yml + content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-proxy + namespace: kube-system + spec: + hostNetwork: true + containers: + - name: kube-proxy + image: ${ hyperkube } + command: + - /hyperkube + - proxy + - --master=http://127.0.0.1:8080 + - --proxy-mode=iptables + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + volumes: + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-host + + - path: /etc/kubernetes/manifests/kube-scheduler.yml + content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-scheduler + namespace: kube-system + spec: + hostNetwork: true + containers: + - name: kube-scheduler + image: ${ hyperkube } + command: + - /hyperkube + - scheduler + - --leader-elect=true + - --master=http://127.0.0.1:8080 + resources: + requests: + cpu: 100m + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10251 + initialDelaySeconds: 15 + timeoutSeconds: 1 + + - path: /etc/logrotate.d/docker-containers + content: | + /var/lib/docker/containers/*/*.log { + rotate 7 + daily + compress + size=1M + missingok + delaycompress + copytruncate + } + + - path: /etc/kubernetes/ssl/ca.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ ca } + + - path: /etc/kubernetes/ssl/k8s-etcd.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_etcd } + + - path: /etc/kubernetes/ssl/k8s-etcd-key.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_etcd_key } + + - path: /etc/kubernetes/ssl/k8s-apiserver.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_apiserver } + + - path: /etc/kubernetes/ssl/k8s-apiserver-key.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_apiserver_key } + diff --git a/provisioning/aws/modules/etcd/ec2.tf b/provisioning/aws/modules/etcd/ec2.tf new file mode 100644 index 0000000..9240b41 --- /dev/null +++ b/provisioning/aws/modules/etcd/ec2.tf @@ -0,0 +1,34 @@ +resource "aws_instance" "etcd" { + count = "${ var.master_node_count }" + + ami = "${ var.ami_id }" + associate_public_ip_address = false + iam_instance_profile = "${ var.instance_profile_name }" + instance_type = "${ var.instance_type }" + key_name = "${ var.key_name }" + + root_block_device { + volume_size = 124 + volume_type = "gp2" + } + + source_dest_check = false + subnet_id = "${ element( split(",", var.subnet_ids_private), 0 ) }" + + tags { + builtWith = "terraform" + KubernetesCluster = "${ var.name }" # used by kubelet's aws provider to determine cluster + kz8s = "${ var.name }" + Name = "etcd${ count.index + 1 }-${ var.name }" + role = "etcd,apiserver" + version = "${ var.kubelet_image_tag }" + visibility = "private" + } + + user_data = "${ element(data.template_file.cloud-config.*.rendered, count.index) }" + vpc_security_group_ids = [ "${ var.etcd_security_group_id }" ] +} + +resource "null_resource" "dummy_dependency" { + depends_on = [ "aws_instance.etcd" ] +} diff --git a/provisioning/aws/modules/etcd/elb.tf b/provisioning/aws/modules/etcd/elb.tf new file mode 100644 index 0000000..33e21f0 --- /dev/null +++ b/provisioning/aws/modules/etcd/elb.tf @@ -0,0 +1,35 @@ +resource "aws_elb" "external" { + name = "kz8s-apiserver-${replace(var.name, "/(.{0,17})(.*)/", "$1")}" + + cross_zone_load_balancing = false + + health_check { + healthy_threshold = 2 + unhealthy_threshold = 2 + timeout = 3 + target = "HTTP:8080/" + interval = 30 + } + + instances = [ "${ aws_instance.etcd.*.id }" ] + idle_timeout = 3600 + + listener { + instance_port = 443 + instance_protocol = "tcp" + lb_port = 443 + lb_protocol = "tcp" + } + + security_groups = [ "${ var.external_elb_security_group_id }" ] + subnets = [ "${ split(",", var.subnet_ids_public) }" ] + + tags { + builtWith = "terraform" + kz8s = "${ var.name }" + Name = "kz8s-apiserver" + role = "apiserver" + visibility = "public" + KubernetesCluster = "${ var.name }" + } +} diff --git a/provisioning/aws/modules/etcd/input.tf b/provisioning/aws/modules/etcd/input.tf new file mode 100644 index 0000000..db88235 --- /dev/null +++ b/provisioning/aws/modules/etcd/input.tf @@ -0,0 +1,25 @@ +variable "ami_id" {} +variable "cluster_domain" {} +variable "kubelet_image_url" {} +variable "kubelet_image_tag" {} +variable "depends_id" {} +variable "dns_service_ip" {} +variable "etcd_security_group_id" {} +variable "external_elb_security_group_id" {} +variable "instance_type" {} +variable "internal_tld" {} +variable "key_name" {} +variable "name" {} +variable "pod_cidr" {} +variable "region" {} +variable "service_cidr" {} +variable "subnet_ids_private" {} +variable "subnet_ids_public" {} +variable "vpc_id" {} +variable "ca" {} +variable "k8s_etcd" {} +variable "k8s_etcd_key" {} +variable "k8s_apiserver" {} +variable "k8s_apiserver_key" {} +variable "instance_profile_name" {} +variable "master_node_count" {} diff --git a/provisioning/aws/modules/etcd/kube-apiserver.yml b/provisioning/aws/modules/etcd/kube-apiserver.yml new file mode 100644 index 0000000..a799bdb --- /dev/null +++ b/provisioning/aws/modules/etcd/kube-apiserver.yml @@ -0,0 +1,58 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kube-apiserver + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: kube-apiserver + image: ${ hyperkube } + command: + - /hyperkube + - apiserver + - --admission-control=LimitRanger + - --admission-control=NamespaceExists + - --admission-control=NamespaceLifecycle + - --admission-control=ResourceQuota + - --admission-control=SecurityContextDeny + - --admission-control=ServiceAccount + - --allow-privileged=true + - --client-ca-file=/etc/kubernetes/ssl/ca.pem + - --cloud-provider=aws + - --etcd-servers=http://etcd.${ internal_tld }:2379 + - --insecure-bind-address=0.0.0.0 + - --secure-port=443 + - --service-account-key-file=/etc/kubernetes/ssl/k8s-apiserver-key.pem + - --service-cluster-ip-range=${ service_cidr } + - --tls-cert-file=/etc/kubernetes/ssl/k8s-apiserver.pem + - --tls-private-key-file=/etc/kubernetes/ssl/k8s-apiserver-key.pem + - --v=2 + livenessProbe: + httpGet: + host: 127.0.0.1 + port: 8080 + path: /healthz + initialDelaySeconds: 15 + timeoutSeconds: 15 + ports: + - containerPort: 443 + hostPort: 443 + name: https + - containerPort: 8080 + hostPort: 8080 + name: local + volumeMounts: + - mountPath: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + readOnly: true + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-host diff --git a/provisioning/aws/modules/etcd/output.tf b/provisioning/aws/modules/etcd/output.tf new file mode 100644 index 0000000..ffd51a1 --- /dev/null +++ b/provisioning/aws/modules/etcd/output.tf @@ -0,0 +1,6 @@ + +#output "depends_id" { value = "${ null_resource.dummy_dependency.id }" } +output "external_elb" { value = "${ aws_elb.external.dns_name }" } +output "internal_ips" { value = "${ join(",", aws_instance.etcd.*.public_ip) }" } + +output "master_ips" { value = ["${ aws_instance.etcd.*.private_ip }"] } diff --git a/provisioning/aws/modules/iam/etcd.tf b/provisioning/aws/modules/iam/etcd.tf new file mode 100644 index 0000000..03673b1 --- /dev/null +++ b/provisioning/aws/modules/iam/etcd.tf @@ -0,0 +1,57 @@ +resource "aws_iam_role" "master" { + name = "master-k8s-${ var.name }" + + assume_role_policy = </dev/null; do sleep 5.2; done; echo "✓" +} + +echo "❤ Polling for cluster life - this could take a minute or more" + +_retry "❤ Waiting for DNS to resolve for ${ELB}" ping -c1 "${ELB}" +_retry "❤ Curling apiserver external elb" curl --insecure --silent "https://${ELB}" +_retry "❤ Trying to connect to cluster with kubectl" kubectl cluster-info + +kubectl cluster-info +sleep 2 # FIXME: Maybe API was up, but scheduling wasn't quite up? diff --git a/azure/azure.tf b/provisioning/azure/azure.tf similarity index 95% rename from azure/azure.tf rename to provisioning/azure/azure.tf index 2623b2d..78f6c81 100644 --- a/azure/azure.tf +++ b/provisioning/azure/azure.tf @@ -10,7 +10,7 @@ resource "azurerm_storage_account" "cncf" { # * azurerm_storage_account.cncf: name can only consist of lowercase letters # and numbers, and must be between 3 and 24 characters long FIXME: # storage_account name must be globally unique - name = "${ var.name }cncfdemo" + name = "${ var.name }x" resource_group_name = "${ var.name }" location = "${ var.location }" account_type = "Standard_LRS" @@ -27,6 +27,5 @@ resource "azurerm_availability_set" "cncf" { name = "${ var.name }" resource_group_name = "${ var.name }" location = "${ var.location }" - } diff --git a/azure/docs/azure_app_endpoints.png b/provisioning/azure/docs/azure_app_endpoints.png similarity index 100% rename from azure/docs/azure_app_endpoints.png rename to provisioning/azure/docs/azure_app_endpoints.png diff --git a/azure/docs/azure_app_registration.png b/provisioning/azure/docs/azure_app_registration.png similarity index 100% rename from azure/docs/azure_app_registration.png rename to provisioning/azure/docs/azure_app_registration.png diff --git a/azure/docs/guid_from_oauth_endpoint.png b/provisioning/azure/docs/guid_from_oauth_endpoint.png similarity index 100% rename from azure/docs/guid_from_oauth_endpoint.png rename to provisioning/azure/docs/guid_from_oauth_endpoint.png diff --git a/azure/docs/key_generation_copy_me.png b/provisioning/azure/docs/key_generation_copy_me.png similarity index 100% rename from azure/docs/key_generation_copy_me.png rename to provisioning/azure/docs/key_generation_copy_me.png diff --git a/azure/docs/research.md b/provisioning/azure/docs/research.md similarity index 100% rename from azure/docs/research.md rename to provisioning/azure/docs/research.md diff --git a/azure/docs/research.org b/provisioning/azure/docs/research.org similarity index 99% rename from azure/docs/research.org rename to provisioning/azure/docs/research.org index a5b2eae..6de72a4 100644 --- a/azure/docs/research.org +++ b/provisioning/azure/docs/research.org @@ -188,7 +188,7 @@ az network lb show --name TestLoadBalancer --resource-group deploy -o table #+BEGIN_SRC -openssl x509 -in .cfssl/k8s-admin.pem -noout -text +openssl x509 -in .cfssl/k8s_admin.pem -noout -text openssl x509 -in .cfssl/k8s-apiserver.pem -noout -text diff --git a/azure/docs/web_api_application_type.png b/provisioning/azure/docs/web_api_application_type.png similarity index 100% rename from azure/docs/web_api_application_type.png rename to provisioning/azure/docs/web_api_application_type.png diff --git a/azure/init-cfssl b/provisioning/azure/init-cfssl similarity index 92% rename from azure/init-cfssl rename to provisioning/azure/init-cfssl index e880777..9bd9e49 100755 --- a/azure/init-cfssl +++ b/provisioning/azure/init-cfssl @@ -87,7 +87,7 @@ function generate { _chmod $CN - tar -cf $CN.tar ca.pem $CN.pem ${CN}-key.pem + #tar -cf $CN.tar ca.pem $CN.pem ${CN}-key.pem } mkdir -p $OUTDIR && cd $OUTDIR @@ -105,5 +105,7 @@ generate k8s-etcd client-server "etcd.${INTERNAL_TLD},etcd1.${INTERNAL_TLD},etcd generate k8s-worker client "${DEFAULT_HOSTS}" # TODO: fix cert provisioning hacks -tar -rf k8s-apiserver.tar k8s-etcd.pem k8s-etcd-key.pem -tar -rf k8s-worker.tar ca.pem +#tar -rf k8s-apiserver.tar k8s-etcd.pem k8s-etcd-key.pem +#tar -rf k8s-worker.tar ca.pem +#bzip2 k8s-apiserver.tar +#bzip2 k8s-worker.tar diff --git a/provisioning/azure/input.tf b/provisioning/azure/input.tf new file mode 100644 index 0000000..66aeb3b --- /dev/null +++ b/provisioning/azure/input.tf @@ -0,0 +1,36 @@ +variable "name" { default = "azure" } + +variable "internal_tld" { default = "azure.cncf.demo" } +variable "data_dir" { default = "/cncf/data/azure" } + +# Azure Cloud Specific Settings +variable "location" { default = "westus" } +variable "vpc_cidr" { default = "10.0.0.0/16" } + +# VM Image and size +variable "admin_username" { default = "cncf"} +variable "image_publisher" { default = "CoreOS" } +variable "image_offer" { default = "CoreOS" } +variable "image_sku" { default = "Stable" } +variable "image_version" { default = "1298.6.0" } +variable "master_vm_size" { default = "Standard_A2" } +variable "worker_vm_size" { default = "Standard_A2" } +variable "bastion_vm_size" { default = "Standard_A2" } + +# Kubernetes +variable "cluster_domain" { default = "cluster.local" } +variable "pod_cidr" { default = "10.2.0.0/16" } +variable "service_cidr" { default = "10.3.0.0/24" } +variable "k8s_service_ip" { default = "10.3.0.1" } +variable "dns_service_ip" { default = "10.3.0.10" } +variable "master_node_count" { default = "3" } +variable "worker_node_count" { default = "3" } +# Autoscaling not supported by Kuberenetes on Azure yet +# variable "worker_node_min" { default = "3" } +# variable "worker_node_max" { default = "5" } + +# Deployment Artifact Versions +# Hyperkube +# Set from https://quay.io/repository/coreos/hyperkube?tab=tags +variable "kubelet_image_url" { default = "quay.io/coreos/hyperkube"} +variable "kubelet_image_tag" { default = "v1.4.7_coreos.0"} diff --git a/provisioning/azure/modules.tf b/provisioning/azure/modules.tf new file mode 100644 index 0000000..2b23120 --- /dev/null +++ b/provisioning/azure/modules.tf @@ -0,0 +1,110 @@ +module "network" { + source = "./modules/network" + name = "${ var.name }" + vpc_cidr = "${ var.vpc_cidr }" + name_servers_file = "${ module.dns.name_servers_file }" + location = "${ var.location }" + } + +module "dns" { + source = "./modules/dns" + name = "${ var.name }" + internal_tld = "${ var.internal_tld }" + master_ips = "${ module.etcd.master_ips }" + master_node_count = "${ var.master_node_count }" + name_servers_file = "${ var.data_dir }/dns" + +} + +module "etcd" { + source = "./modules/etcd" + name = "${ var.name }" + location = "${ var.location }" + admin_username = "${ var.admin_username }" + master_node_count = "${ var.master_node_count }" + master_vm_size = "${ var.master_vm_size }" + image_publisher = "${ var.image_publisher }" + image_offer = "${ var.image_offer }" + image_sku = "${ var.image_sku }" + image_version = "${ var.image_version }" + subnet_id = "${ module.network.subnet_id }" + storage_account = "${ azurerm_storage_account.cncf.name }" + storage_primary_endpoint = "${ azurerm_storage_account.cncf.primary_blob_endpoint }" + storage_container = "${ var.name }" + # storage_container = "${ azurerm_storage_container.cncf.name }" + availability_id = "${ azurerm_availability_set.cncf.id }" + cluster_domain = "${ var.cluster_domain }" + kubelet_image_url = "${ var.kubelet_image_url }" + kubelet_image_tag = "${ var.kubelet_image_tag }" + dns_service_ip = "${ var.dns_service_ip }" + internal_tld = "${ var.internal_tld }" + pod_cidr = "${ var.pod_cidr }" + service_cidr = "${ var.service_cidr }" + k8s_cloud_config = "${file("${ var.data_dir }/azure-config.json")}" + ca = "${file("${ var.data_dir }/.cfssl/ca.pem")}" + k8s_etcd = "${file("${ var.data_dir }/.cfssl/k8s-etcd.pem")}" + k8s_etcd_key = "${file("${ var.data_dir }/.cfssl/k8s-etcd-key.pem")}" + k8s_apiserver = "${file("${ var.data_dir }/.cfssl/k8s-apiserver.pem")}" + k8s_apiserver_key = "${file("${ var.data_dir }/.cfssl/k8s-apiserver-key.pem")}" + data_dir = "${ var.data_dir }" +} + + +module "bastion" { + source = "./modules/bastion" + name = "${ var.name }" + location = "${ var.location }" + bastion_vm_size = "${ var.bastion_vm_size }" + image_publisher = "${ var.image_publisher }" + image_offer = "${ var.image_offer }" + image_sku = "${ var.image_sku }" + image_version = "${ var.image_version }" + admin_username = "${ var.admin_username }" + subnet_id = "${ module.network.subnet_id }" + storage_primary_endpoint = "${ azurerm_storage_account.cncf.primary_blob_endpoint }" + storage_container = "${ azurerm_storage_container.cncf.name }" + availability_id = "${ azurerm_availability_set.cncf.id }" + internal_tld = "${ var.internal_tld }" + data_dir = "${ var.data_dir }" +} + +module "worker" { + source = "./modules/worker" + name = "${ var.name }" + location = "${ var.location }" + admin_username = "${ var.admin_username }" + worker_node_count = "${ var.worker_node_count }" + worker_vm_size = "${ var.worker_vm_size }" + image_publisher = "${ var.image_publisher }" + image_offer = "${ var.image_offer }" + image_sku = "${ var.image_sku }" + image_version = "${ var.image_version }" + subnet_id = "${ module.network.subnet_id }" + storage_account = "${ azurerm_storage_account.cncf.name }" + storage_primary_endpoint = "${ azurerm_storage_account.cncf.primary_blob_endpoint }" + storage_container = "${ azurerm_storage_container.cncf.name }" + availability_id = "${ azurerm_availability_set.cncf.id }" + external_lb = "${ module.etcd.external_lb }" + cluster_domain = "${ var.cluster_domain }" + kubelet_image_url = "${ var.kubelet_image_url }" + kubelet_image_tag = "${ var.kubelet_image_tag }" + dns_service_ip = "${ var.dns_service_ip }" + internal_tld = "${ var.internal_tld }" + k8s_cloud_config = "${file("${ var.data_dir }/azure-config.json")}" + ca = "${file("${ var.data_dir }/.cfssl/ca.pem")}" + k8s_worker = "${file("${ var.data_dir }/.cfssl/k8s-worker.pem")}" + k8s_worker_key = "${file("${ var.data_dir }/.cfssl/k8s-worker-key.pem")}" + data_dir = "${ var.data_dir }" +} + + +module "kubeconfig" { + source = "../kubeconfig" + + admin_key_pem = "${ var.data_dir }/.cfssl/k8s-admin-key.pem" + admin_pem = "${ var.data_dir }/.cfssl/k8s-admin.pem" + ca_pem = "${ var.data_dir }/.cfssl/ca.pem" + data_dir = "${ var.data_dir }" + fqdn_k8s = "${ module.etcd.fqdn_lb }" + name = "${ var.name }" +} diff --git a/azure/modules/bastion/node.tf b/provisioning/azure/modules/bastion/bastion-node.tf similarity index 60% rename from azure/modules/bastion/node.tf rename to provisioning/azure/modules/bastion/bastion-node.tf index 6611d79..d589c2e 100644 --- a/azure/modules/bastion/node.tf +++ b/provisioning/azure/modules/bastion/bastion-node.tf @@ -13,7 +13,7 @@ resource "azurerm_network_interface" "cncf" { ip_configuration { name = "${ var.name }" - subnet_id = "${ var.subnet-id }" + subnet_id = "${ var.subnet_id }" private_ip_address_allocation = "dynamic" public_ip_address_id = "${ azurerm_public_ip.cncf.id }" } @@ -22,45 +22,45 @@ resource "azurerm_network_interface" "cncf" { resource "azurerm_virtual_machine" "cncf" { name = "${ var.name }" location = "${ var.location }" - availability_set_id = "${ var.availability-id }" + availability_set_id = "${ var.availability_id }" resource_group_name = "${ var.name }" network_interface_ids = ["${azurerm_network_interface.cncf.id}"] - vm_size = "${ var.bastion-vm-size }" + vm_size = "${ var.bastion_vm_size }" storage_image_reference { - publisher = "${ var.image-publisher }" - offer = "${ var.image-offer }" - sku = "${ var.image-sku }" - version = "${ var.image-version}" + publisher = "${ var.image_publisher }" + offer = "${ var.image_offer }" + sku = "${ var.image_sku }" + version = "${ var.image_version}" } storage_os_disk { name = "disk2" - vhd_uri = "${ var.storage-primary-endpoint }${ var.storage-container }/disk2.vhd" + vhd_uri = "${ var.storage_primary_endpoint }${ var.storage_container }/disk2.vhd" caching = "ReadWrite" create_option = "FromImage" } os_profile { computer_name = "hostname" - admin_username = "${ var.admin-username }" + admin_username = "${ var.admin_username }" admin_password = "Password1234!" - custom_data = "${ data.template_file.user-data.rendered }" + custom_data = "${ data.template_file.bastion-user-data.rendered }" #custom_data = "${file("${path.module}/user-data2.yml")}" } os_profile_linux_config { disable_password_authentication = true ssh_keys { - path = "/home/${ var.admin-username }/.ssh/authorized_keys" - key_data = "${file("/cncf/data/.ssh/id_rsa.pub")}" + path = "/home/${ var.admin_username }/.ssh/authorized_keys" + key_data = "${file("${ var.data_dir }/.ssh/id_rsa.pub")}" } } } -data "template_file" "user-data" { - template = "${ file( "${ path.module }/user-data.yml" )}" +data "template_file" "bastion-user-data" { + template = "${ file( "${ path.module }/bastion-user-data.yml" )}" vars { - internal-tld = "${ var.internal-tld }" + internal_tld = "${ var.internal_tld }" } } diff --git a/azure/modules/bastion/user-data.yml b/provisioning/azure/modules/bastion/bastion-user-data.yml similarity index 94% rename from azure/modules/bastion/user-data.yml rename to provisioning/azure/modules/bastion/bastion-user-data.yml index db22516..7e00497 100644 --- a/azure/modules/bastion/user-data.yml +++ b/provisioning/azure/modules/bastion/bastion-user-data.yml @@ -6,7 +6,7 @@ coreos: reboot-strategy: etcd-lock etcd2: - discovery-srv: ${ internal-tld } + discovery-srv: ${ internal_tld } proxy: on units: diff --git a/provisioning/azure/modules/bastion/input.tf b/provisioning/azure/modules/bastion/input.tf new file mode 100644 index 0000000..9dc1a42 --- /dev/null +++ b/provisioning/azure/modules/bastion/input.tf @@ -0,0 +1,18 @@ +variable "name" {} +variable "location" {} +variable "bastion_vm_size" {} +variable "image_publisher" {} +variable "image_offer" {} +variable "image_sku" {} +variable "image_version" {} +variable "admin_username" {} +variable "internal_tld" {} +variable "subnet_id" {} +variable "availability_id" {} +variable "storage_container" {} +variable "storage_primary_endpoint" {} +variable "data_dir" {} + +# variable "allow_ssh_cidr" {} +# variable "security_group_id" {} +# variable "subnet_ids" {} diff --git a/provisioning/azure/modules/bastion/output.tf b/provisioning/azure/modules/bastion/output.tf new file mode 100644 index 0000000..2f8f327 --- /dev/null +++ b/provisioning/azure/modules/bastion/output.tf @@ -0,0 +1,2 @@ +output "bastion_ip" { value = "${azurerm_public_ip.cncf.ip_address}" } +output "bastion_fqdn" { value = "${azurerm_public_ip.cncf.fqdn}" } diff --git a/azure/modules/dns/dns.tf b/provisioning/azure/modules/dns/dns.tf similarity index 78% rename from azure/modules/dns/dns.tf rename to provisioning/azure/modules/dns/dns.tf index c04064d..28206e3 100644 --- a/azure/modules/dns/dns.tf +++ b/provisioning/azure/modules/dns/dns.tf @@ -1,5 +1,5 @@ resource "azurerm_dns_zone" "cncf" { - name = "${ var.internal-tld }" + name = "${ var.internal_tld }" resource_group_name = "${ var.name }" } @@ -15,7 +15,7 @@ resource "null_resource" "dns_dig" { # filename = "/dev/null" command = < ${ var.name-servers-file }.${ count.index }.ip +dig +short ${ element(azurerm_dns_zone.cncf.name_servers,count.index) } > ${ var.name_servers_file }.${ count.index }.ip EOF } } @@ -28,10 +28,10 @@ resource "null_resource" "dns_gen" { # collect them all into a csv # wait for them all to appear sleep 4 -cat ${ var.name-servers-file }.*.ip \ +cat ${ var.name_servers_file }.*.ip \ | sed -n -e 'H;$${x;s/\n/,/g;s/^,//;p;}' \ | tr -d '\n' \ -> ${ var.name-servers-file} +> ${ var.name_servers_file} EOF } } @@ -42,19 +42,19 @@ resource "azurerm_dns_a_record" "A-etcd" { resource_group_name = "${ var.name }" ttl = "300" records = [ - "${ var.master-ips }" + "${ var.master_ips }" ] } resource "azurerm_dns_a_record" "A-etcds" { - count = "${ length(var.master-ips) }" + count = "${ var.master_node_count }" name = "etcd${ count.index+1 }" zone_name = "${azurerm_dns_zone.cncf.name}" resource_group_name = "${ var.name }" ttl = "300" records = [ - "${ element(var.master-ips, count.index) }" + "${ element(var.master_ips, count.index) }" ] } @@ -63,17 +63,17 @@ resource "azurerm_dns_a_record" "A-master" { zone_name = "${azurerm_dns_zone.cncf.name}" resource_group_name = "${ var.name }" ttl = "300" - records = [ "${ var.master-ips }" ] + records = [ "${ var.master_ips }" ] } resource "azurerm_dns_a_record" "A-masters" { - count = "${ length(var.master-ips) }" + count = "${ var.master_node_count }" name = "master${ count.index+1 }" zone_name = "${azurerm_dns_zone.cncf.name}" resource_group_name = "${ var.name }" ttl = "300" records = [ - "${ element(var.master-ips, count.index) }" + "${ element(var.master_ips, count.index) }" ] } @@ -87,21 +87,21 @@ resource "azurerm_dns_srv_record" "etcd-client-tcp" { priority = 0 weight = 0 port = 2379 - target = "etcd1.${ var.internal-tld }" + target = "etcd1.${ var.internal_tld }" } record { priority = 0 weight = 0 port = 2379 - target = "etcd2.${ var.internal-tld }" + target = "etcd2.${ var.internal_tld }" } record { priority = 0 weight = 0 port = 2379 - target = "etcd3.${ var.internal-tld }" + target = "etcd3.${ var.internal_tld }" } } @@ -116,21 +116,21 @@ resource "azurerm_dns_srv_record" "etcd-server-tcp" { priority = 0 weight = 0 port = 2380 - target = "etcd1.${ var.internal-tld }" + target = "etcd1.${ var.internal_tld }" } record { priority = 0 weight = 0 port = 2380 - target = "etcd2.${ var.internal-tld }" + target = "etcd2.${ var.internal_tld }" } record { priority = 0 weight = 0 port = 2380 - target = "etcd3.${ var.internal-tld }" + target = "etcd3.${ var.internal_tld }" } } diff --git a/provisioning/azure/modules/dns/input.tf b/provisioning/azure/modules/dns/input.tf new file mode 100644 index 0000000..09d721f --- /dev/null +++ b/provisioning/azure/modules/dns/input.tf @@ -0,0 +1,5 @@ +variable "internal_tld" {} +variable "name" {} +variable "name_servers_file" {} +variable "master_ips" { type = "list" } +variable "master_node_count" {} diff --git a/provisioning/azure/modules/dns/output.tf b/provisioning/azure/modules/dns/output.tf new file mode 100644 index 0000000..74d6c33 --- /dev/null +++ b/provisioning/azure/modules/dns/output.tf @@ -0,0 +1,4 @@ +# output "depends_id" { value = "${null_resource.dummy_dependency.id}" } +output "internal_name_servers" { value = "${ azurerm_dns_zone.cncf.name_servers }" } +output "internal_zone_id" { value = "${ azurerm_dns_zone.cncf.zone_id }" } +output "name_servers_file" { value = "${ var.name_servers_file }" } diff --git a/provisioning/azure/modules/etcd/etcd-cloud-config.tf b/provisioning/azure/modules/etcd/etcd-cloud-config.tf new file mode 100644 index 0000000..da52e0d --- /dev/null +++ b/provisioning/azure/modules/etcd/etcd-cloud-config.tf @@ -0,0 +1,70 @@ +provider "gzip" { + compressionlevel = "BestCompression" +} + +resource "gzip_me" "kube-apiserver" { + input = "${ data.template_file.kube_apiserver.rendered }" +} +resource "gzip_me" "k8s_cloud_config" { + input = "${ var.k8s_cloud_config }" +} + +resource "gzip_me" "ca" { + input = "${ var.ca }" +} + +resource "gzip_me" "k8s_etcd" { + input = "${ var.k8s_etcd }" +} + +resource "gzip_me" "k8s_etcd_key" { + input = "${ var.k8s_etcd_key }" +} + +resource "gzip_me" "k8s_apiserver" { + input = "${ var.k8s_apiserver }" +} + +resource "gzip_me" "k8s_apiserver_key" { + input = "${ var.k8s_apiserver_key }" +} + +data "template_file" "kube_apiserver" { + template = "${ file( "${ path.module }/kube-apiserver.yml" )}" + vars { + internal_tld = "${ var.internal_tld }" + service_cidr = "${ var.service_cidr }" + hyperkube = "${ var.kubelet_image_url }:${ var.kubelet_image_tag }" + kubelet_image_url = "${ var.kubelet_image_url }" + kubelet_image_tag = "${ var.kubelet_image_tag }" + } +} + +data "template_file" "etcd_cloud_config" { + count = "${ var.master_node_count }" + template = "${ file( "${ path.module }/etcd-cloud-config.yml" )}" + + vars { + # bucket = "${ var.s3_bucket }" + cluster_domain = "${ var.cluster_domain }" + cluster-token = "etcd-cluster-${ var.name }" + dns_service_ip = "${ var.dns_service_ip }" + fqdn = "etcd${ count.index + 1 }.${ var.internal_tld }" + hostname = "etcd${ count.index + 1 }" + kubelet_image_url = "${ var.kubelet_image_url }" + kubelet_image_tag = "${ var.kubelet_image_tag }" + internal_tld = "${ var.internal_tld }" + pod_cidr = "${ var.pod_cidr }" + location = "${ var.location }" + service_cidr = "${ var.service_cidr }" + k8s_cloud_config = "${ gzip_me.k8s_cloud_config.output }" + ca = "${ gzip_me.ca.output }" + k8s_etcd = "${ gzip_me.k8s_etcd.output }" + k8s_etcd_key = "${ gzip_me.k8s_etcd_key.output }" + k8s_apiserver = "${ gzip_me.k8s_apiserver.output }" + k8s_apiserver_key = "${ gzip_me.k8s_apiserver_key.output }" + k8s_apiserver_yml = "${ gzip_me.kube-apiserver.output }" + node-ip = "${ element(azurerm_network_interface.cncf.*.private_ip_address, count.index) }" + + } +} diff --git a/provisioning/azure/modules/etcd/etcd-cloud-config.yml b/provisioning/azure/modules/etcd/etcd-cloud-config.yml new file mode 100644 index 0000000..3b0fb83 --- /dev/null +++ b/provisioning/azure/modules/etcd/etcd-cloud-config.yml @@ -0,0 +1,235 @@ +#cloud-config + +--- +coreos: + + etcd2: + advertise-client-urls: http://${ fqdn }:2379 + # cert-file: /etc/kubernetes/ssl/k8s-etcd.pem + debug: true + discovery-srv: ${ internal_tld } + initial-advertise-peer-urls: https://${ fqdn }:2380 + initial-cluster-state: new + initial-cluster-token: ${ cluster-token } + # key-file: /etc/kubernetes/ssl/k8s-etcd-key.pem + listen-client-urls: http://${ node-ip }:2379,http://127.0.0.1:2379 + listen-peer-urls: https://${ node-ip }:2380 + name: ${ hostname } + peer-trusted-ca-file: /etc/kubernetes/ssl/ca.pem + peer-client-cert-auth: true + peer-cert-file: /etc/kubernetes/ssl/k8s-etcd.pem + peer-key-file: /etc/kubernetes/ssl/k8s-etcd-key.pem + + units: + - name: etcd2.service + command: start + + - name: docker.service + command: start + drop-ins: + - name: overlay.conf + content: | + [Service] + Environment="DOCKER_OPTS=--storage-driver=overlay" + + - name: kubelet.service + command: start + content: | + [Unit] + ConditionFileIsExecutable=/usr/lib/coreos/kubelet-wrapper + [Service] + Environment="KUBELET_IMAGE_URL=${ kubelet_image_url }" + Environment="KUBELET_IMAGE_TAG=${ kubelet_image_tag }" + Environment="RKT_OPTS=\ + --volume dns,kind=host,source=/etc/resolv.conf \ + --mount volume=dns,target=/etc/resolv.conf \ + --volume rkt,kind=host,source=/opt/bin/host-rkt \ + --mount volume=rkt,target=/usr/bin/rkt \ + --volume var-lib-rkt,kind=host,source=/var/lib/rkt \ + --mount volume=var-lib-rkt,target=/var/lib/rkt \ + --volume stage,kind=host,source=/tmp \ + --mount volume=stage,target=/tmp \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log" + ExecStartPre=/usr/bin/mkdir -p /var/log/containers + ExecStartPre=/usr/bin/mkdir -p /var/lib/kubelet + ExecStartPre=/usr/bin/mount --bind /var/lib/kubelet /var/lib/kubelet + ExecStartPre=/usr/bin/mount --make-shared /var/lib/kubelet + ExecStart=/usr/lib/coreos/kubelet-wrapper \ + --allow-privileged=true \ + --api-servers=http://127.0.0.1:8080 \ + --cloud-provider=azure \ + --cloud-config=/etc/kubernetes/ssl/azure-config.json \ + --cluster-dns=${ dns_service_ip } \ + --cluster_domain=${ cluster_domain } \ + --config=/etc/kubernetes/manifests + Restart=always + RestartSec=5 + [Install] + WantedBy=multi-user.target + + update: + reboot-strategy: etcd-lock + +write-files: + - path: /opt/bin/host-rkt + permissions: 0755 + owner: root:root + content: | + #!/bin/sh + exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "$@" + + - path: /etc/kubernetes/manifests/kube-apiserver.yml + encoding: "gzip+base64" + content: | + ${ k8s_apiserver_yml } + + - path: /etc/kubernetes/manifests/kube-controller-manager.yml + content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-controller-manager + namespace: kube-system + spec: + hostNetwork: true + containers: + - name: kube-controller-manager + image: ${ kubelet_image_url }:${ kubelet_image_tag } + command: + - /hyperkube + - controller-manager + - --cloud-provider=azure + - --cloud-config=/etc/kubernetes/ssl/azure-config.json + - --leader-elect=true + - --master=http://127.0.0.1:8080 + - --root-ca-file=/etc/kubernetes/ssl/ca.pem + - --service-account-private-key-file=/etc/kubernetes/ssl/k8s-apiserver-key.pem + resources: + requests: + cpu: 200m + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10252 + initialDelaySeconds: 15 + timeoutSeconds: 1 + volumeMounts: + - mountPath: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + readOnly: true + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-host + + - path: /etc/kubernetes/manifests/kube-proxy.yml + content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-proxy + namespace: kube-system + spec: + hostNetwork: true + containers: + - name: kube-proxy + image: ${ kubelet_image_url }:${ kubelet_image_tag } + command: + - /hyperkube + - proxy + - --master=http://127.0.0.1:8080 + - --proxy-mode=iptables + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + volumes: + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-host + + - path: /etc/kubernetes/manifests/kube-scheduler.yml + content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-scheduler + namespace: kube-system + spec: + hostNetwork: true + containers: + - name: kube-scheduler + image: ${ kubelet_image_url }:${ kubelet_image_tag } + command: + - /hyperkube + - scheduler + - --leader-elect=true + - --master=http://127.0.0.1:8080 + resources: + requests: + cpu: 100m + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10251 + initialDelaySeconds: 15 + timeoutSeconds: 1 + + - path: /etc/logrotate.d/docker-containers + content: | + /var/lib/docker/containers/*/*.log { + rotate 7 + daily + compress + size=1M + missingok + delaycompress + copytruncate + } + + - path: /etc/kubernetes/ssl/ca.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ ca } + + - path: /etc/kubernetes/ssl/k8s-etcd.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_etcd } + + - path: /etc/kubernetes/ssl/k8s-etcd-key.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_etcd_key } + + - path: /etc/kubernetes/ssl/k8s-apiserver.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_apiserver } + + - path: /etc/kubernetes/ssl/k8s-apiserver-key.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_apiserver_key } + + - path: /etc/kubernetes/ssl/azure-config.json + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_cloud_config } diff --git a/azure/modules/etcd/load-balancer.tf b/provisioning/azure/modules/etcd/etcd-load-balancer.tf similarity index 97% rename from azure/modules/etcd/load-balancer.tf rename to provisioning/azure/modules/etcd/etcd-load-balancer.tf index 1266605..e0fad61 100644 --- a/azure/modules/etcd/load-balancer.tf +++ b/provisioning/azure/modules/etcd/etcd-load-balancer.tf @@ -3,7 +3,7 @@ resource "azurerm_public_ip" "cncf" { location = "${ var.location }" resource_group_name = "${ var.name }" public_ip_address_allocation = "static" - domain_name_label = "k8s${ var.name }" + domain_name_label = "k8s-${ var.name }" } resource "azurerm_lb" "cncf" { diff --git a/azure/modules/etcd/nodes.tf b/provisioning/azure/modules/etcd/etcd-nodes.tf similarity index 60% rename from azure/modules/etcd/nodes.tf rename to provisioning/azure/modules/etcd/etcd-nodes.tf index b96a4fc..1bb779c 100644 --- a/azure/modules/etcd/nodes.tf +++ b/provisioning/azure/modules/etcd/etcd-nodes.tf @@ -1,12 +1,12 @@ resource "azurerm_network_interface" "cncf" { - count = "${ var.master-node-count }" + count = "${ var.master_node_count }" name = "etcd-interface${ count.index + 1 }" location = "${ var.location }" resource_group_name = "${ var.name }" ip_configuration { name = "etcd-nic${ count.index + 1 }" - subnet_id = "${ var.subnet-id }" + subnet_id = "${ var.subnet_id }" private_ip_address_allocation = "dynamic" # private_ip_address = "${ element( split(",", var.etcd-ips), count.index ) }" load_balancer_backend_address_pools_ids = ["${ azurerm_lb_backend_address_pool.cncf.id }"] @@ -14,40 +14,40 @@ resource "azurerm_network_interface" "cncf" { } resource "azurerm_virtual_machine" "cncf" { - count = "${ var.master-node-count }" + count = "${ var.master_node_count }" name = "etcd-master${ count.index + 1 }" location = "${ var.location }" - availability_set_id = "${ var.availability-id }" + availability_set_id = "${ var.availability_id }" resource_group_name = "${ var.name }" network_interface_ids = ["${ element(azurerm_network_interface.cncf.*.id, count.index) }"] - vm_size = "${ var.master-vm-size }" + vm_size = "${ var.master_vm_size }" storage_image_reference { - publisher = "${ var.image-publisher }" - offer = "${ var.image-offer }" - sku = "${ var.image-sku }" - version = "${ var.image-version}" + publisher = "${ var.image_publisher }" + offer = "${ var.image_offer }" + sku = "${ var.image_sku }" + version = "${ var.image_version}" } storage_os_disk { name = "etcd-disks${ count.index + 1 }" - vhd_uri = "${ var.storage-primary-endpoint }${ var.storage-container }/etcd-vhd${ count.index + 1 }.vhd" + vhd_uri = "${ var.storage_primary_endpoint }${ var.storage_container }/etcd-vhd${ count.index + 1 }.vhd" caching = "ReadWrite" create_option = "FromImage" } os_profile { computer_name = "etcd-master${ count.index + 1 }" - admin_username = "${ var.admin-username }" + admin_username = "${ var.admin_username }" admin_password = "Password1234!" - custom_data = "${ element(data.template_file.cloud-config.*.rendered, count.index) }" + custom_data = "${ element(data.template_file.etcd_cloud_config.*.rendered, count.index) }" } os_profile_linux_config { disable_password_authentication = true ssh_keys { - path = "/home/${ var.admin-username }/.ssh/authorized_keys" - key_data = "${file("/cncf/data/.ssh/id_rsa.pub")}" + path = "/home/${ var.admin_username }/.ssh/authorized_keys" + key_data = "${file("${ var.data_dir }/.ssh/id_rsa.pub")}" } } } diff --git a/provisioning/azure/modules/etcd/input.tf b/provisioning/azure/modules/etcd/input.tf new file mode 100644 index 0000000..a7a2bb6 --- /dev/null +++ b/provisioning/azure/modules/etcd/input.tf @@ -0,0 +1,30 @@ +variable "location" {} +variable "subnet_id" {} +variable "name" {} +variable "master_vm_size" {} +variable "master_node_count" {} +variable "image_publisher" {} +variable "image_offer" {} +variable "image_sku" {} +variable "image_version" {} +variable "availability_id" {} +variable "storage_account" {} +variable "storage_primary_endpoint" {} +variable "storage_container" {} +variable "cluster_domain" {} +variable "dns_service_ip" {} +variable "internal_tld" {} +variable "pod_cidr" {} +variable "service_cidr" {} +variable "admin_username" {} +variable "kubelet_image_url" {} +variable "kubelet_image_tag" {} +variable "k8s_cloud_config" {} +# variable "etcd_security_group_id" {} +# variable "external_elb_security_group_id" {} +variable "ca" {} +variable "k8s_etcd" {} +variable "k8s_etcd_key" {} +variable "k8s_apiserver" {} +variable "k8s_apiserver_key" {} +variable "data_dir" {} diff --git a/provisioning/azure/modules/etcd/kube-apiserver.yml b/provisioning/azure/modules/etcd/kube-apiserver.yml new file mode 100644 index 0000000..c85a1f5 --- /dev/null +++ b/provisioning/azure/modules/etcd/kube-apiserver.yml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kube-apiserver + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: kube-apiserver + image: ${ kubelet_image_url }:${ kubelet_image_tag } + command: + - /hyperkube + - apiserver + - --admission-control=LimitRanger + - --admission-control=NamespaceExists + - --admission-control=NamespaceLifecycle + - --admission-control=ResourceQuota + - --admission-control=SecurityContextDeny + - --admission-control=ServiceAccount + - --allow-privileged=true + - --client-ca-file=/etc/kubernetes/ssl/ca.pem + - --cloud-provider=azure + - --cloud-config=/etc/kubernetes/ssl/azure-config.json + - --etcd-servers=http://etcd.${ internal_tld }:2379 + - --insecure-bind-address=0.0.0.0 + - --secure-port=443 + - --service-account-key-file=/etc/kubernetes/ssl/k8s-apiserver-key.pem + - --service-cluster-ip-range=${ service_cidr } + - --tls-cert-file=/etc/kubernetes/ssl/k8s-apiserver.pem + - --tls-private-key-file=/etc/kubernetes/ssl/k8s-apiserver-key.pem + - --v=2 + livenessProbe: + httpGet: + host: 127.0.0.1 + port: 8080 + path: /healthz + initialDelaySeconds: 15 + timeoutSeconds: 15 + ports: + - containerPort: 443 + hostPort: 443 + name: https + - containerPort: 8080 + hostPort: 8080 + name: local + volumeMounts: + - mountPath: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + readOnly: true + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/ssl + name: ssl-certs-kubernetes + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-host diff --git a/provisioning/azure/modules/etcd/output.tf b/provisioning/azure/modules/etcd/output.tf new file mode 100644 index 0000000..19f77f4 --- /dev/null +++ b/provisioning/azure/modules/etcd/output.tf @@ -0,0 +1,3 @@ +output "external_lb" { value = "${azurerm_lb_backend_address_pool.cncf.id }" } +output "fqdn_lb" { value = "${azurerm_public_ip.cncf.fqdn}" } +output "master_ips" { value = ["${ azurerm_network_interface.cncf.*.private_ip_address }"] } diff --git a/provisioning/azure/modules/network/input.tf b/provisioning/azure/modules/network/input.tf new file mode 100644 index 0000000..6ff315e --- /dev/null +++ b/provisioning/azure/modules/network/input.tf @@ -0,0 +1,4 @@ +variable "vpc_cidr" {} +variable "name" {} +variable "name_servers_file" {} +variable "location" {} diff --git a/provisioning/azure/modules/network/output.tf b/provisioning/azure/modules/network/output.tf new file mode 100644 index 0000000..daad37c --- /dev/null +++ b/provisioning/azure/modules/network/output.tf @@ -0,0 +1 @@ +output "subnet_id" { value = "${ azurerm_subnet.cncf.id }" } diff --git a/azure/modules/vpc/vpc.tf b/provisioning/azure/modules/network/virtual_network.tf similarity index 56% rename from azure/modules/vpc/vpc.tf rename to provisioning/azure/modules/network/virtual_network.tf index 926f90c..4cd5910 100644 --- a/azure/modules/vpc/vpc.tf +++ b/provisioning/azure/modules/network/virtual_network.tf @@ -1,19 +1,26 @@ -#resource "null_resource" "dummy_dependency" { -# depends_on = [ -# "aws_vpc.main", -# "aws_nat_gateway.nat" -# ] -#} +resource "azurerm_network_security_group" "cncf" { + name = "${ var.name }" + location = "${ var.location}" + resource_group_name = "${ var.name }" +} +resource "azurerm_subnet" "cncf" { + name = "${ var.name }" + resource_group_name = "${ var.name }" + virtual_network_name = "${azurerm_virtual_network.cncf.name}" + address_prefix = "10.0.10.0/24" + route_table_id = "${ azurerm_route_table.cncf.id }" + +} resource "azurerm_virtual_network" "cncf" { name = "${ var.name }" resource_group_name = "${ var.name }" - address_space = ["${ var.cidr }"] + address_space = ["${ var.vpc_cidr }"] location = "${ var.location }" dns_servers = [ - "${ element(split( ",", file(var.name-servers-file) ),0) }", - "${ element(split( ",", file(var.name-servers-file) ),1) }", + "${ element(split( ",", file(var.name_servers_file) ),0) }", + "${ element(split( ",", file(var.name_servers_file) ),1) }", "8.8.8.8" ] # getting dns servers in list form was difficult @@ -25,15 +32,6 @@ resource "azurerm_virtual_network" "cncf" { # dns_servers.1: "" => "13.107.24.9" # dns_servers.2: "" => "64.4.48.9" # dns_servers.3: "" => "13.107.160.9" - - # tags { - #builtWith = "terraform" - #KubernetesCluster = "${ var.name }" - #z8s = "${ var.name }" - #Name = "kz8s-${ var.name }" - #version = "${ var.hyperkube-tag }" - #visibility = "private,public" - #} } resource "azurerm_route_table" "cncf" { diff --git a/provisioning/azure/modules/worker/input.tf b/provisioning/azure/modules/worker/input.tf new file mode 100644 index 0000000..aa275e2 --- /dev/null +++ b/provisioning/azure/modules/worker/input.tf @@ -0,0 +1,25 @@ +variable "location" {} +variable "subnet_id" {} +variable "name" {} +variable "worker_vm_size" {} +variable "worker_node_count" {} +variable "image_publisher" {} +variable "image_offer" {} +variable "image_sku" {} +variable "image_version" {} +variable "storage_account" {} +variable "storage_primary_endpoint" {} +variable "storage_container" {} +variable "availability_id" {} +variable "external_lb" {} +variable "cluster_domain" {} +variable "dns_service_ip" {} +variable "internal_tld" {} +variable "admin_username" {} +variable "kubelet_image_url" {} +variable "kubelet_image_tag" {} +variable "k8s_cloud_config" {} +variable "ca" {} +variable "k8s_worker" {} +variable "k8s_worker_key" {} +variable "data_dir" {} diff --git a/provisioning/azure/modules/worker/worker-cloud-config.tf b/provisioning/azure/modules/worker/worker-cloud-config.tf new file mode 100644 index 0000000..79a589a --- /dev/null +++ b/provisioning/azure/modules/worker/worker-cloud-config.tf @@ -0,0 +1,37 @@ +provider "gzip" { + compressionlevel = "BestCompression" +} + +resource "gzip_me" "k8s_cloud_config" { + input = "${ var.k8s_cloud_config }" +} + +resource "gzip_me" "ca" { + input = "${ var.ca }" +} + +resource "gzip_me" "k8s_worker" { + input = "${ var.k8s_worker }" +} + +resource "gzip_me" "k8s_worker_key" { + input = "${ var.k8s_worker_key }" +} + +data "template_file" "worker_cloud_config" { + template = "${ file( "${ path.module }/worker-cloud-config.yml" )}" + + vars { + cluster_domain = "${ var.cluster_domain }" + dns_service_ip = "${ var.dns_service_ip }" + kubelet_image_url = "${ var.kubelet_image_url }" + kubelet_image_tag = "${ var.kubelet_image_tag }" + internal_tld = "${ var.internal_tld }" + location = "${ var.location }" + k8s_cloud_config = "${ gzip_me.k8s_cloud_config.output }" + ca = "${ gzip_me.ca.output }" + k8s_worker = "${ gzip_me.k8s_worker.output }" + k8s_worker_key = "${ gzip_me.k8s_worker_key.output }" + } +} + diff --git a/provisioning/azure/modules/worker/worker-cloud-config.yml b/provisioning/azure/modules/worker/worker-cloud-config.yml new file mode 100644 index 0000000..e739bb4 --- /dev/null +++ b/provisioning/azure/modules/worker/worker-cloud-config.yml @@ -0,0 +1,165 @@ +#cloud-config +--- +coreos: + + etcd2: + discovery-srv: ${ internal_tld } + peer-trusted-ca-file: /etc/kubernetes/ssl/ca.pem + peer-client-cert-auth: true + peer-cert-file: /etc/kubernetes/ssl/k8s-worker.pem + peer-key-file: /etc/kubernetes/ssl/k8s-worker-key.pem + proxy: on + + units: + - name: etcd2.service + command: start + + - name: docker.service + command: start + + - name: kubelet.service + command: start + content: | + [Unit] + ConditionFileIsExecutable=/usr/lib/coreos/kubelet-wrapper + [Service] + Environment="KUBELET_IMAGE_URL=${ kubelet_image_url }" + Environment="KUBELET_IMAGE_TAG=${ kubelet_image_tag }" + Environment="RKT_OPTS=\ + --volume dns,kind=host,source=/etc/resolv.conf \ + --mount volume=dns,target=/etc/resolv.conf \ + --volume rkt,kind=host,source=/opt/bin/host-rkt \ + --mount volume=rkt,target=/usr/bin/rkt \ + --volume var-lib-rkt,kind=host,source=/var/lib/rkt \ + --mount volume=var-lib-rkt,target=/var/lib/rkt \ + --volume stage,kind=host,source=/tmp \ + --mount volume=stage,target=/tmp \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log" + ExecStartPre=/usr/bin/mkdir -p /var/log/containers + ExecStartPre=/usr/bin/mkdir -p /var/lib/kubelet + ExecStartPre=/usr/bin/mount --bind /var/lib/kubelet /var/lib/kubelet + ExecStartPre=/usr/bin/mount --make-shared /var/lib/kubelet + ExecStart=/usr/lib/coreos/kubelet-wrapper \ + --allow-privileged=true \ + --api-servers=http://master.${ internal_tld }:8080 \ + --cloud-provider=azure \ + --cloud-config=/etc/kubernetes/ssl/azure-config.json \ + --cluster-dns=${ dns_service_ip } \ + --cluster_domain=${ cluster_domain } \ + --config=/etc/kubernetes/manifests \ + --kubeconfig=/etc/kubernetes/kubeconfig.yml \ + --register-node=true \ + --tls-cert-file=/etc/kubernetes/ssl/k8s-worker.pem \ + --tls-private-key-file=/etc/kubernetes/ssl/k8s-worker-key.pem + Restart=always + RestartSec=5 + [Install] + WantedBy=multi-user.target + + update: + reboot-strategy: etcd-lock + +write-files: + - path: /opt/bin/host-rkt + permissions: 0755 + owner: root:root + content: | + #!/bin/sh + exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "$@" + + - path: /etc/kubernetes/kubeconfig.yml + content: | + apiVersion: v1 + kind: Config + clusters: + - name: local + cluster: + certificate-authority: /etc/kubernetes/ssl/ca.pem + users: + - name: kubelet + user: + client-certificate: /etc/kubernetes/ssl/k8s-worker.pem + client-key: /etc/kubernetes/ssl/k8s-worker-key.pem + contexts: + - context: + cluster: local + user: kubelet + name: kubelet-context + current-context: kubelet-context + + - path: /etc/kubernetes/manifests/kube-proxy.yml + content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-proxy + namespace: kube-system + spec: + hostNetwork: true + containers: + - name: kube-proxy + image: ${ kubelet_image_url }:${ kubelet_image_tag } + command: + - /hyperkube + - proxy + - --kubeconfig=/etc/kubernetes/kubeconfig.yml + - --master=https://master.${ internal_tld } + - --proxy-mode=iptables + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/ssl/certs + name: "ssl-certs" + - mountPath: /etc/kubernetes/kubeconfig.yml + name: "kubeconfig" + readOnly: true + - mountPath: /etc/kubernetes/ssl + name: "etc-kube-ssl" + readOnly: true + volumes: + - name: "ssl-certs" + hostPath: + path: "/usr/share/ca-certificates" + - name: "kubeconfig" + hostPath: + path: "/etc/kubernetes/kubeconfig.yml" + - name: "etc-kube-ssl" + hostPath: + path: "/etc/kubernetes/ssl" + + - path: /etc/logrotate.d/docker-containers + content: | + /var/lib/docker/containers/*/*.log { + rotate 7 + daily + compress + size=1M + missingok + delaycompress + copytruncate + } + + - path: /etc/kubernetes/ssl/ca.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ ca } + + - path: /etc/kubernetes/ssl/k8s-worker.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_worker } + + - path: /etc/kubernetes/ssl/k8s-worker-key.pem + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_worker_key } + + - path: /etc/kubernetes/ssl/azure-config.json + permissions: "0644" + encoding: "gzip+base64" + content: | + ${ k8s_cloud_config } diff --git a/provisioning/azure/modules/worker/worker-nodes.tf b/provisioning/azure/modules/worker/worker-nodes.tf new file mode 100644 index 0000000..7b28b44 --- /dev/null +++ b/provisioning/azure/modules/worker/worker-nodes.tf @@ -0,0 +1,51 @@ +resource "azurerm_network_interface" "cncf" { + count = "${ var.worker_node_count }" + name = "worker-interface${ count.index + 1 }" + location = "${ var.location }" + resource_group_name = "${ var.name }" + + ip_configuration { + name = "worker-nic${ count.index + 1 }" + subnet_id = "${ var.subnet_id }" + private_ip_address_allocation = "dynamic" + } +} + +resource "azurerm_virtual_machine" "cncf" { + count = "${ var.worker_node_count }" + name = "worker-node${ count.index + 1 }" + location = "${ var.location }" + availability_set_id = "${ var.availability_id }" + resource_group_name = "${ var.name }" + network_interface_ids = ["${ element(azurerm_network_interface.cncf.*.id, count.index) }"] + vm_size = "${ var.worker_vm_size }" + + storage_image_reference { + publisher = "${ var.image_publisher }" + offer = "${ var.image_offer }" + sku = "${ var.image_sku }" + version = "${ var.image_version}" + } + + storage_os_disk { + name = "worker-disks${ count.index + 1 }" + vhd_uri = "${ var.storage_primary_endpoint }${ var.storage_container }/worker-vhd${ count.index + 1 }.vhd" + caching = "ReadWrite" + create_option = "FromImage" + } + + os_profile { + computer_name = "worker-node${ count.index + 1 }" + admin_username = "${ var.admin_username }" + admin_password = "Password1234!" + custom_data = "${ element(data.template_file.worker_cloud_config.*.rendered, count.index) }" + } + + os_profile_linux_config { + disable_password_authentication = true + ssh_keys { + path = "/home/${ var.admin_username }/.ssh/authorized_keys" + key_data = "${file("${ var.data_dir }/.ssh/id_rsa.pub")}" + } + } +} diff --git a/provisioning/azure/output.tf b/provisioning/azure/output.tf new file mode 100644 index 0000000..8306c86 --- /dev/null +++ b/provisioning/azure/output.tf @@ -0,0 +1,8 @@ +output "fqdn_k8s" { value = "${ module.etcd.fqdn_lb}" } +output "bastion_ip" { value = "${ module.bastion.bastion_ip}" } +output "bastion_fqdn" { value = "${ module.bastion.bastion_fqdn}" } +output "k8s_admin" { value = "${ k8s_admin}"} +# fixme for use outside container +output "ssh_key_setup" { value = "eval $(ssh-agent) ; ssh-add ${ var.data_dir }/.ssh/id_rsa"} +output "ssh_via_bastion" { value = "ssh -At ${ var.admin_username }@${ module.bastion.bastion_fqdn } ssh ${ var.admin_username }@etcd1.${ var.internal_tld }"} +output "kubeconfig" { value = "${ module.kubeconfig.kubeconfig }"} diff --git a/azure/readme.org b/provisioning/azure/readme.org similarity index 95% rename from azure/readme.org rename to provisioning/azure/readme.org index 4cec32b..3d78322 100644 --- a/azure/readme.org +++ b/provisioning/azure/readme.org @@ -25,7 +25,7 @@ docker run -ti -v $(pwd)/data/$DEPLOYNAME:/cncf/data --env-file ./data/azure.env #+NAME: 3 minute deploy #+BEGIN_SRC output bastion-fqdn = bastion201703230324.westus.cloudapp.azure.com -fqdn-k8s = k8s201703230324.westus.cloudapp.azure.com +fqdn_k8s = k8s201703230324.westus.cloudapp.azure.com ssh-key-setup = eval $(ssh-agent) ; ssh-add /cncf/data/.ssh/id_rsa ssh-via-bastion = ssh -At cncf@bastion201703230324.westus.cloudapp.azure.com ssh cncf@master1.cncf.demo @@ -61,18 +61,18 @@ Adding var-name=value to ./data/terraform.tfvars will allow you to override many #+BEGIN_SRC shell name= "azure" location= "westus" -internal-tld= "cncf.demo" -master-node-count= "3" -worker-node-count= "3" -master-vm-size= "Standard_A2" -worker-vm-size= "Standard_A2" -bastion-vm-size= "Standard_A2" -kubelet-image-url= "quay.io/coreos/hyperkube" -kubelet-image-tag= "v1.4.7_coreos.0" -image-publisher= "CoreOS" -image-offer= "CoreOS" -image-sku= "Stable" -image-version= "1298.6.0" +internal_tld= "cncf.demo" +master_node_count= "3" +worker_node_count= "3" +master_vm_size= "Standard_A2" +worker_vm_size= "Standard_A2" +bastion_vm_size= "Standard_A2" +kubelet_aci= "quay.io/coreos/hyperkube" +kubelet_image_tag= "v1.4.7_coreos.0" +image_publisher= "CoreOS" +image_offer= "CoreOS" +image_sku= "Stable" +image_version= "1298.6.0" #+END_SRC * Next steps diff --git a/azure/runme b/provisioning/azure/runme similarity index 100% rename from azure/runme rename to provisioning/azure/runme diff --git a/provisioning/azure/servicePrincipalProfile.json b/provisioning/azure/servicePrincipalProfile.json new file mode 100644 index 0000000..e69de29 diff --git a/provisioning/azure/ssl-ssh-cloud.tf b/provisioning/azure/ssl-ssh-cloud.tf new file mode 100644 index 0000000..d02a3a0 --- /dev/null +++ b/provisioning/azure/ssl-ssh-cloud.tf @@ -0,0 +1,70 @@ +#Gen Certs and SSH KeyPair +resource "null_resource" "ssl_ssh_cloud_gen" { + + provisioner "local-exec" { + command = < ${ var.data_dir }/azure-config.json +{ + "aadClientId": "$${ARM_CLIENT_ID}", + "aadClientSecret": "$${ARM_CLIENT_SECRET}", + "tenantId": "$${ARM_TENANT_ID}", + "subscriptionId": "$${ARM_SUBSCRIPTION_ID}", + "resourceGroup": "${ var.name }", + "location": "${ var.location }", + "subnetName": "${ var.name }", + "securityGroupName": "${ var.name }", + "vnetName": "${ var.name }", + "routeTableName": "${ var.name }", + "primaryAvailabilitySetName": "${ var.name }" +} +JSON +COMMAND + } + + provisioner "local-exec" { + when = "destroy" + on_failure = "continue" + command = </dev/null; do sleep 5.2; done; echo "✓" +} + +echo "❤ Polling for cluster life - this could take a minute or more" + +_retry "❤ Waiting for DNS to resolve for ${ELB}" getent hosts "${ELB}" +_retry "❤ Curling apiserver external elb" curl --insecure --silent "https://${ELB}" +_retry "❤ Trying to connect to cluster with kubectl" kubectl cluster-info + +kubectl cluster-info +sleep 2 # FIXME: Maybe API was up, but scheduling wasn't quite up? diff --git a/provisioning/cross-cloud/cloud.tf b/provisioning/cross-cloud/cloud.tf new file mode 100644 index 0000000..4003bf8 --- /dev/null +++ b/provisioning/cross-cloud/cloud.tf @@ -0,0 +1,57 @@ +module "aws" { + source = "../aws" + name = "${ var.name }-aws" + internal_tld = "${ var.name }-aws.cncf.demo" + data_dir = "${ var.data_dir }/aws" +} + +module "azure" { + source = "../azure" + name = "${ var.name }azure" + internal_tld = "${ var.name }-azure.cncf.demo" + data_dir = "${ var.data_dir }/azure" +} + +module "packet" { + source = "../packet" + name = "${ var.name }-packet" + data_dir = "${ var.data_dir }/packet" + packet_project_id = "${ var.packet_project_id }" +} + +module "gce" { + source = "../gce" + name = "${ var.name }-gce" + data_dir = "${ var.data_dir }/gce" +} + +module "gke" { + source = "../gke" + name = "${ var.name }-gke" + data_dir = "${ var.data_dir}/gke" +} + + +resource "null_resource" "kubeconfig" { + + provisioner "local-exec" { + command = < ${ var.data-dir }/azure-config.json +# { +# "aadClientId": "$${ARM_CLIENT_ID}", +# "aadClientSecret": "$${ARM_CLIENT_SECRET}", +# "tenantId": "$${ARM_TENANT_ID}", +# "subscriptionId": "$${ARM_SUBSCRIPTION_ID}", +# "resourceGroup": "${ var.name }", +# "location": "${ var.location }", +# "subnetName": "${ var.name }", +# "securityGroupName": "${ var.name }", +# "vnetName": "${ var.name }", +# "routeTableName": "${ var.name }", +# "primaryAvailabilitySetName": "${ var.name }" +# } +# JSON +# COMMAND +# } + +# provisioner "local-exec" { +# when = "destroy" +# on_failure = "continue" +# command = <ca-csr.json +echo "$(ca-config)" >ca-config.json + +# generate ca +cfssl gencert -initca ca-csr.json | cfssljson -bare ca - +_chmod ca + +# generate keys and certs +generate k8s-admin client-server "${DEFAULT_HOSTS}" +generate k8s-apiserver client-server "${DEFAULT_HOSTS},${K8S_SERVICE_IP},master.${NAME}.${DOMAIN},endpoint.${NAME}.${DOMAIN}" +generate k8s-etcd client-server "*.c.${PROJECT}.internal,test-master1.c.${PROJECT}.internal,test-master2.c.${PROJECT}.internal,test-master3.c.${PROJECT}.internal" +generate k8s-worker client "${DEFAULT_HOSTS}" + +# Fix cert provisioning hacks +tar -rf k8s-apiserver.tar k8s-etcd.pem k8s-etcd-key.pem +tar -rf k8s-worker.tar ca.pem diff --git a/provisioning/gce/input.tf b/provisioning/gce/input.tf new file mode 100644 index 0000000..ceee0fc --- /dev/null +++ b/provisioning/gce/input.tf @@ -0,0 +1,28 @@ +variable "name" { default = "test" } +variable "internal_tld" { default = "cncf.demo" } +variable "master_node_count" { default = "3" } +variable "worker_node_count" { default = "3" } +#variable "master_vm_size" { default = "Standard_A2" } +#variable "worker-vm-size" { default = "Standard_A2" } +#variable "bastion-vm-size" { default = "Standard_A2" } +# Set from https://quay.io/repository/coreos/hyperkube?tab=tags +variable "kubelet_image_url" { default = "quay.io/coreos/hyperkube"} +variable "kubelet_image_tag" { default = "v1.4.7_coreos.0"} +#variable "image-publisher" { default = "CoreOS" } +#variable "image-offer" { default = "CoreOS" } +#variable "image-sku" { default = "Stable" } +#variable "image-version" { default = "1298.6.0" } +variable "region" { default = "us-central1" } +variable "zone" { default = "us-central1-a" } +variable "project" { default = "test-163823" } +variable "cluster_domain" { default = "cluster.local" } +# variable "admin_username" { default = "cncf"} +variable "cidr" { default = "10.0.0.0/16" } +variable "pod_cidr" { default = "10.2.0.0/16" } +variable "service_cidr" { default = "10.3.0.0/24" } +variable "k8s_service_ip" { default = "10.3.0.1" } +variable "dns_service_ip" { default = "10.3.0.10" } +# variable "allow-ssh-cidr" { default = "0.0.0.0/0" } +variable "data_dir" { default = "/cncf/data" } +# variable "name-servers-file" { default = "google_dns_zone"} +variable "domain" { default = "cncf.ci" } diff --git a/azure/keypair.tf b/provisioning/gce/keypair.tf similarity index 72% rename from azure/keypair.tf rename to provisioning/gce/keypair.tf index 45f3c01..336ca49 100644 --- a/azure/keypair.tf +++ b/provisioning/gce/keypair.tf @@ -3,8 +3,8 @@ resource "null_resource" "sshkey_gen" { provisioner "local-exec" { command = < ${ var.etcd_discovery } +EOF + } + + provisioner "local-exec" { + when = "destroy" + on_failure = "continue" + command = < "${ var.data_dir }/ca.pem" +echo "${ base64decode(google_container_cluster.cncf.master_auth.0.client_certificate) }" > "${ var.data_dir }/k8s-admin.pem" +echo "${ base64decode(google_container_cluster.cncf.master_auth.0.client_key) }" > "${ var.data_dir }/k8s-admin-key.pem" +LOCAL_EXEC + } +} diff --git a/provisioning/gke/modules/cluster/input.tf b/provisioning/gke/modules/cluster/input.tf new file mode 100644 index 0000000..aa57ce7 --- /dev/null +++ b/provisioning/gke/modules/cluster/input.tf @@ -0,0 +1,14 @@ +variable "name" {} +variable "region" {} +variable "zone" {} +variable "project" {} +variable "node_count" {} +variable "network" {} +variable "subnetwork" {} +variable "node_version" {} +variable "master_user" {} +variable "master_password" {} +variable "node_pool_count" {} +variable "vm_size" {} +variable "data_dir" {} + diff --git a/provisioning/gke/modules/cluster/node-pool.tf b/provisioning/gke/modules/cluster/node-pool.tf new file mode 100644 index 0000000..1eacac8 --- /dev/null +++ b/provisioning/gke/modules/cluster/node-pool.tf @@ -0,0 +1,7 @@ +resource "google_container_node_pool" "cncf" { + name = "${ var.name }" + project = "${ var.project }" + zone = "${ var.zone }" + cluster = "${google_container_cluster.cncf.name}" + initial_node_count = "${ var.node_pool_count }" +} diff --git a/provisioning/gke/modules/cluster/output.tf b/provisioning/gke/modules/cluster/output.tf new file mode 100644 index 0000000..bf4a453 --- /dev/null +++ b/provisioning/gke/modules/cluster/output.tf @@ -0,0 +1 @@ +output "fqdn_k8s" { value = "${ google_container_cluster.cncf.endpoint }" } diff --git a/provisioning/gke/modules/vpc/gce-subnet.tf b/provisioning/gke/modules/vpc/gce-subnet.tf new file mode 100644 index 0000000..49ac226 --- /dev/null +++ b/provisioning/gke/modules/vpc/gce-subnet.tf @@ -0,0 +1,6 @@ +resource "google_compute_subnetwork" "cncf" { + name = "${ var.name }" + ip_cidr_range = "${ var.cidr }" + network = "${ google_compute_network.cncf.self_link }" + region = "${ var.region }" +} diff --git a/provisioning/gke/modules/vpc/input.tf b/provisioning/gke/modules/vpc/input.tf new file mode 100644 index 0000000..e523c3e --- /dev/null +++ b/provisioning/gke/modules/vpc/input.tf @@ -0,0 +1,3 @@ +variable "cidr" {} +variable "name" {} +variable "region" {} diff --git a/provisioning/gke/modules/vpc/output.tf b/provisioning/gke/modules/vpc/output.tf new file mode 100644 index 0000000..e69de29 diff --git a/provisioning/gke/modules/vpc/vpc.tf b/provisioning/gke/modules/vpc/vpc.tf new file mode 100644 index 0000000..7394480 --- /dev/null +++ b/provisioning/gke/modules/vpc/vpc.tf @@ -0,0 +1,4 @@ +resource "google_compute_network" "cncf" { + name = "${ var.name }" + auto_create_subnetworks = "false" +} diff --git a/provisioning/gke/output.tf b/provisioning/gke/output.tf new file mode 100644 index 0000000..2234222 --- /dev/null +++ b/provisioning/gke/output.tf @@ -0,0 +1 @@ +output "kubeconfig" { value = "${ module.kubeconfig.kubeconfig }"} diff --git a/provisioning/kubeconfig/input.tf b/provisioning/kubeconfig/input.tf new file mode 100644 index 0000000..19ab85c --- /dev/null +++ b/provisioning/kubeconfig/input.tf @@ -0,0 +1,6 @@ +variable "admin_key_pem" {} +variable "admin_pem" {} +variable "ca_pem" {} +variable "fqdn_k8s" {} +variable "name" {} +variable "data_dir" {} diff --git a/provisioning/kubeconfig/kubeconfig.tf b/provisioning/kubeconfig/kubeconfig.tf new file mode 100644 index 0000000..0278337 --- /dev/null +++ b/provisioning/kubeconfig/kubeconfig.tf @@ -0,0 +1,41 @@ +data "template_file" "kubeconfig" { + template = <ca-csr.json +echo "$(ca-config)" >ca-config.json + +# generate ca +cfssl gencert -initca ca-csr.json | cfssljson -bare ca - +_chmod ca + +# generate keys and certs +generate k8s-admin client-server "${DEFAULT_HOSTS}" +generate k8s-apiserver client-server "${DEFAULT_HOSTS},${K8S_SERVICE_IP},master.${INTERNAL_TLD},endpoint.${INTERNAL_TLD}" +generate k8s-etcd client-server "etcd.${INTERNAL_TLD},etcd1.${INTERNAL_TLD},etcd2.${INTERNAL_TLD},etcd3.${INTERNAL_TLD}" +generate k8s-worker client "${DEFAULT_HOSTS}" + +# TODO: fix cert provisioning hacks +#tar -rf k8s-apiserver.tar k8s-etcd.pem k8s-etcd-key.pem +#tar -rf k8s-worker.tar ca.pem +#bzip2 k8s-apiserver.tar +#bzip2 k8s-worker.tar diff --git a/provisioning/packet/input.tf b/provisioning/packet/input.tf new file mode 100644 index 0000000..34d76e5 --- /dev/null +++ b/provisioning/packet/input.tf @@ -0,0 +1,34 @@ +variable "name" { default = "packet" } + +# Set with env TF_VAR_packet_project_id +variable "packet_project_id" {} # required for now +# https://www.packet.net/locations/ +variable "packet_facility" { default = "sjc1" } +variable "packet_billing_cycle" { default = "hourly" } +variable "packet_operating_system" { default = "coreos_stable" } +variable "packet_master_device_plan" { default = "baremetal_0" } +variable "packet_worker_device_plan" { default = "baremetal_0" } + +variable "domain" { default = "cncf.ci" } +variable "data_dir" { default = "/cncf/data/packet" } + +# VM Image and size +variable "admin_username" { default = "core"} + +# Kubernetes +variable "cluster_domain" { default = "cluster.local" } +variable "pod_cidr" { default = "10.2.0.0/16" } +variable "service_cidr" { default = "10.3.0.0/24" } +variable "k8s_service_ip" { default = "10.3.0.1" } +variable "dns_service_ip" { default = "10.3.0.10" } +variable "master_node_count" { default = "3" } +variable "worker_node_count" { default = "3" } +# Autoscaling not supported by Kuberenetes on Azure yet +# variable "worker_node_min" { default = "3" } +# variable "worker_node_max" { default = "5" } + +# Deployment Artifact Versions +# Hyperkube +# Set from https://quay.io/repository/coreos/hyperkube +variable "kubelet_image_url" { default = "quay.io/coreos/hyperkube"} +variable "kubelet_image_tag" { default = "v1.6.2_coreos.0"} diff --git a/provisioning/packet/modules.tf b/provisioning/packet/modules.tf new file mode 100644 index 0000000..e74f243 --- /dev/null +++ b/provisioning/packet/modules.tf @@ -0,0 +1,97 @@ +# module "network" { +# source = "./modules/network" +# name = "${ var.name }" +# cidr = "${ var.vpc_cidr }" +# name_servers_file = "${ module.dns.name_servers_file }" +# location = "${ var.location }" +# } + +module "dns" { + source = "./modules/dns" + name = "${ var.name }" + master_ips = "${ module.etcd.master_ips }" + public_master_ips = "${ module.etcd.public_master_ips }" + public_worker_ips = "${ module.worker.public_worker_ips }" + master_node_count = "${ var.master_node_count }" + worker_node_count = "${ var.worker_node_count }" + domain = "${ var.domain }" +} + +module "etcd" { + source = "./modules/etcd" + name = "${ var.name }" + etcd_discovery = "${ var.data_dir }/etcd" + master_node_count = "${ var.master_node_count }" + packet_project_id = "${ var.packet_project_id }" + packet_facility = "${ var.packet_facility }" + packet_billing_cycle = "${ var.packet_billing_cycle }" + packet_operating_system = "${ var.packet_operating_system }" + packet_master_device_plan = "${ var.packet_master_device_plan }" + kubelet_image_url = "${ var.kubelet_image_url }" + kubelet_image_tag = "${ var.kubelet_image_tag }" + dns_service_ip = "${ var.dns_service_ip }" + cluster_domain = "${ var.cluster_domain }" + internal_tld = "${ var.name }.${ var.domain }" + pod_cidr = "${ var.pod_cidr }" + service_cidr = "${ var.service_cidr }" + ca = "${file("${ var.data_dir }/.cfssl/ca.pem")}" + k8s_etcd = "${file("${ var.data_dir }/.cfssl/k8s-etcd.pem")}" + k8s_etcd_key = "${file("${ var.data_dir }/.cfssl/k8s-etcd-key.pem")}" + k8s_apiserver = "${file("${ var.data_dir }/.cfssl/k8s-apiserver.pem")}" + k8s_apiserver_key = "${file("${ var.data_dir }/.cfssl/k8s-apiserver-key.pem")}" + data_dir = "${ var.data_dir }" +} + +# module "bastion" { +# source = "./modules/bastion" +# name = "${ var.name }" +# location = "${ var.location }" +# bastion_vm_size = "${ var.bastion_vm_size }" +# image_publisher = "${ var.image_publisher }" +# image_offer = "${ var.image_offer }" +# image_sku = "${ var.image_sku }" +# image_version = "${ var.image_version }" +# admin_username = "${ var.admin_username }" +# subnet_id = "${ module.network.subnet_id }" +# storage_primary_endpoint = "${ azurerm_storage_account.cncf.primary_blob_endpoint }" +# storage_container = "${ azurerm_storage_container.cncf.name }" +# availability_id = "${ azurerm_availability_set.cncf.id }" +# internal_tld = "${ var.internal_tld }" +# data_dir = "${ var.data_dir }" +# } + +module "worker" { + source = "./modules/worker" + name = "${ var.name }" + worker_node_count = "${ var.worker_node_count }" + packet_project_id = "${ var.packet_project_id }" + packet_facility = "${ var.packet_facility }" + packet_billing_cycle = "${ var.packet_billing_cycle }" + packet_operating_system = "${ var.packet_operating_system }" + packet_worker_device_plan = "${ var.packet_worker_device_plan }" + kubelet_image_url = "${ var.kubelet_image_url }" + kubelet_image_tag = "${ var.kubelet_image_tag }" + dns_service_ip = "${ var.dns_service_ip }" + cluster_domain = "${ var.cluster_domain }" + internal_tld = "${ var.name }.${ var.domain }" + pod_cidr = "${ var.pod_cidr }" + service_cidr = "${ var.service_cidr }" + ca = "${file("${ var.data_dir }/.cfssl/ca.pem")}" + k8s_etcd = "${file("${ var.data_dir }/.cfssl/k8s-etcd.pem")}" + k8s_etcd_key = "${file("${ var.data_dir }/.cfssl/k8s-etcd-key.pem")}" + k8s_worker = "${file("${ var.data_dir }/.cfssl/k8s-worker.pem")}" + k8s_worker_key = "${file("${ var.data_dir }/.cfssl/k8s-worker-key.pem")}" + etcd_discovery = "${ var.data_dir }/etcd" + data_dir = "${ var.data_dir }" +} + +module "kubeconfig" { + source = "../kubeconfig" + + admin_key_pem = "${ var.data_dir }/.cfssl/k8s-admin-key.pem" + admin_pem = "${ var.data_dir }/.cfssl/k8s-admin.pem" + ca_pem = "${ var.data_dir }/.cfssl/ca.pem" + data_dir = "${ var.data_dir }" + fqdn_k8s = "endpoint.${ var.name }.${ var.domain }" + name = "${ var.name }" +} diff --git a/provisioning/packet/modules/dns/dns.tf b/provisioning/packet/modules/dns/dns.tf new file mode 100644 index 0000000..452a345 --- /dev/null +++ b/provisioning/packet/modules/dns/dns.tf @@ -0,0 +1,87 @@ +# domain - (Required) The domain to add the record to +# name - (Required) The name of the record +# value - (Required) The value of the record +# type - (Required) The type of the record +# ttl - (Optional) The TTL of the record +# priority - (Optional) The priority of the record - only useful for some record typesi + +resource "dnsimple_record" "A-etcd" { + count = "${ var.master_node_count }" + name = "etcd.${ var.name }" + value = "${ element(var.master_ips, count.index) }" + type = "A" + ttl = "${ var.record_ttl }" + domain = "${ var.domain }" +} + +resource "dnsimple_record" "A-etcds" { + count = "${ var.master_node_count }" + name = "etcd${ count.index+1 }.${ var.name }" + value = "${ element(var.master_ips, count.index) }" + type = "A" + ttl = "${ var.record_ttl }" + domain = "${ var.domain }" +} + +resource "dnsimple_record" "A-masters" { + count = "${ var.master_node_count }" + name = "master${ count.index+1 }.${ var.name }" + value = "${ element(var.master_ips, count.index) }" + type = "A" + ttl = "${ var.record_ttl }" + domain = "${ var.domain }" +} + +resource "dnsimple_record" "A-master" { + count = "${ var.master_node_count }" + name = "master.${ var.name }" + value = "${ element(var.master_ips, count.index) }" + type = "A" + ttl = "${ var.record_ttl }" + domain = "${ var.domain }" +} + +resource "dnsimple_record" "A-public-endpoint" { + count = "${ var.master_node_count }" + name = "endpoint.${ var.name }" + value = "${ element(var.public_master_ips, count.index) }" + type = "A" + ttl = "${ var.record_ttl }" + domain = "${ var.domain }" +} + +resource "dnsimple_record" "A-public-masters" { + count = "${ var.master_node_count }" + name = "master${ count.index + 1 }.public.${ var.name }" + value = "${ element(var.public_master_ips, count.index) }" + type = "A" + ttl = "${ var.record_ttl }" + domain = "${ var.domain }" +} + +resource "dnsimple_record" "A-public-workers" { + count = "${ var.worker_node_count }" + name = "worker${ count.index + 1 }.${ var.name }" + value = "${ element(var.public_worker_ips, count.index) }" + type = "A" + ttl = "${ var.record_ttl }" + domain = "${ var.domain }" +} + +# resource "dnsimple_record" "etcd-client-tcp" { +# count = "${ var.master_node_count }" +# name = "_etcd-client._tcp.${ var.name }" +# value = "etcd${ count.index+1 }.${ var.name }.${ var.domain }." +# type = "SRV" +# ttl = "${ var.record_ttl }" +# domain = "${ var.domain }" +# } + +# resource "dnsimple_record" "etcd-server-tcp" { +# count = "${ var.master_node_count }" +# name = "_etcd-server-ssl._tcp.${ var.name }" +# value = "etcd${ count.index+1 }.${ var.name }.${ var.domain }." +# type = "SRV" +# ttl = "${ var.record_ttl }" +# domain = "${ var.domain }" +# } diff --git a/provisioning/packet/modules/dns/input.tf b/provisioning/packet/modules/dns/input.tf new file mode 100644 index 0000000..5cf4803 --- /dev/null +++ b/provisioning/packet/modules/dns/input.tf @@ -0,0 +1,8 @@ +variable "name" {} +variable "master_ips" { type = "list" } +variable "public_master_ips" { type = "list" } +variable "public_worker_ips" { type = "list" } +variable "master_node_count" {} +variable "worker_node_count" {} +variable "domain" {} +variable "record_ttl" { default = "60" } diff --git a/provisioning/packet/modules/dns/output.tf b/provisioning/packet/modules/dns/output.tf new file mode 100644 index 0000000..e69de29 diff --git a/provisioning/packet/modules/etcd/discovery.tf b/provisioning/packet/modules/etcd/discovery.tf new file mode 100644 index 0000000..a321d09 --- /dev/null +++ b/provisioning/packet/modules/etcd/discovery.tf @@ -0,0 +1,18 @@ +#Get Discovery URL +resource "null_resource" "discovery_gen" { + + provisioner "local-exec" { + command = < ${ var.etcd_discovery } +EOF + } + + provisioner "local-exec" { + when = "destroy" + on_failure = "continue" + command = <