Merge pull request #1 from mattermost/node-rotator

Rotator tool to accelerate k8s cluster upgrades and node rotations.

Author: stylianosrigas

.gitignore

@@ -0,0 +1,3 @@
+coverage.out
+.vscode
+build/_output/

Makefile

@@ -0,0 +1,54 @@
+# Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+# See LICENSE.txt for license information.
+
+## Docker Build Versions
+DOCKER_BUILD_IMAGE = golang:1.15.8
+DOCKER_BASE_IMAGE = alpine:3.13
+
+# Variables
+GO = go
+APP := rotator
+APPNAME := node-rotator
+ROTATOR_IMAGE ?= mattermost/node-rotator:test
+
+################################################################################
+
+export GO111MODULE=on
+
+all: check-style unittest fmt
+
+.PHONY: check-style
+check-style: govet
+	@echo Checking for style guide compliance
+
+.PHONY: vet
+govet:
+	@echo Running govet
+	$(GO) vet ./...
+	@echo Govet success
+
+.PHONY: fmt
+fmt: ## Run go fmt against code
+	go fmt ./...
+
+.PHONY: unittest
+unittest:
+	$(GO) test ./... -v -covermode=count -coverprofile=coverage.out
+
+# Build for distribution
+.PHONY: build
+build:
+	@echo Building Mattermost Rotator
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 $(GO) build -gcflags all=-trimpath=$(PWD) -asmflags all=-trimpath=$(PWD) -a -installsuffix cgo -o build/_output/bin/$(APP)  ./cmd/$(APP)
+
+
+# Builds the docker image
+.PHONY: build-image
+build-image:
+	@echo Building Rotator Docker Image
+	docker build \
+	--build-arg DOCKER_BUILD_IMAGE=$(DOCKER_BUILD_IMAGE) \
+	--build-arg DOCKER_BASE_IMAGE=$(DOCKER_BASE_IMAGE) \
+	. -f build/Dockerfile \
+	-t $(ROTATOR_IMAGE) \
+	--no-cache

README.md

@@ -1 +1,93 @@
-# node-rotator
+# Rotator 
+
+Rotator is a tool meant to smooth and accelerate k8s cluster upgrades and node rotations. It offers automation on autoscaling group recognition and flexility on options such as, how fast to rotate nodes, drain retries, waiting time between rotations and drains as well as mater/worker node separation. 
+
+## How to use
+
+The rotator tool can be imported either as a go package and used in any app that needs to rotate/detach/drain k8s nodes or used as a cli tool, where the rotator server can accept rotation requests.
+
+### Import as Go package
+
+To import as a Go package both the [rotator]("github.com/mattermost/rotator/rotator") and the [model]("github.com/mattermost/rotator/model") should be imported. 
+
+The rotator should be called with a cluster object like the one bellow:
+
+```golang
+clusterRotator := rotatorModel.Cluster{
+		ClusterID:            <The id of the cluster to rotate nodes>, (string)
+		MaxScaling:           <Maximum number of nodes to rotate in each rotation>, (int)
+		RotateMasters:        <if master nodes should be rotated>, (bool)
+		RotateWorkers:        <if worker nodes should be rotated>, (bool)
+		MaxDrainRetries:      <max number of retries when a node drain fails>, (int)
+		EvictGracePeriod:     <pod evict grace period>, (int)
+		WaitBetweenRotations: <wait between each rotation of groups of nodes defined by MaxScaling in seconds>, (int)
+		WaitBetweenDrains:    <wait between each node drain in a group of nodes>, (int)
+		ClientSet:            <k8s clientset>, (*kubernetes.Clientset)
+	}
+```
+
+Calling the `InitRotateCluster` function of the rotator package with the defined clusterRotator object is all is needed to rotate a cluster. Example can be seen bellow:
+
+```golang
+rotatorMetadata, err = rotator.InitRotateCluster(&clusterRotator, rotatorMetadata, logger)
+	if err != nil {
+		cluster.ProvisionerMetadataKops.RotatorRequest.Status = rotatorMetadata
+		return err
+	}
+```
+
+where 
+
+```golang
+rotatorMetadata = &rotator.RotatorMetadata{}
+```
+
+The rotator returns metadata that in case of rotation failure include information of ASGs pending rotation. This metadata can be passed back to the InitRotateCluster and the rotator will resume from where it left. 
+
+
+### Use Rotator as CLI tool
+
+The Rotator can be used as a docker image or as a local server. 
+
+#### Building
+
+Simply run the following:
+
+```bash
+go install ./cmd/rotator
+alias cloud='$HOME/go/bin/rotator'
+```
+
+#### Running
+
+Run the server with:
+
+```bash
+rotator server
+```
+
+In a different terminal/window, to rotate a cluster:
+```bash
+rotator cluster rotate --cluster <cluster_id> --rotate-workers --rotate-masters --wait-between-rotations 30 --wait-between-drains 60 --max-scaling 4 --evict-grace-period 30
+```
+
+You will get a response like this one:
+```bash
+[{
+    "ClusterID": "<cluster_id>",
+    "MaxScaling": 4,
+    "RotateMasters": true,
+    "RotateWorkers": true,
+    "MaxDrainRetries": 10,
+    "EvictGracePeriod": 30,
+    "WaitBetweenRotations": 30,
+    "WaitBetweenDrains": 30,
+    "ClientSet": null
+}
+```
+
+### Other Setup
+
+For the rotator to run access to both the AWS account and the K8s cluster is required to be able to do actions such as, `DescribeInstances`, `DetachInstances`, `TerminateInstances`, `DescribeAutoScalingGroups`, as well as `drain`, `kill`, `evict` pods, etc.
+
+The relevant AWS Access and Secret key pair should be exported and k8s access should be provided via a passed clientset or a locally exported k8s config. 

api/api.go

@@ -0,0 +1,67 @@
+package api
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/mattermost/rotator/model"
+	rotator "github.com/mattermost/rotator/rotator"
+)
+
+// Register registers the API endpoints on the given router.
+func Register(rootRouter *mux.Router, context *Context) {
+	apiRouter := rootRouter.PathPrefix("/api").Subrouter()
+
+	initCluster(apiRouter, context)
+}
+
+// initCluster registers RDS cluster endpoints on the given router.
+func initCluster(apiRouter *mux.Router, context *Context) {
+	addContext := func(handler contextHandlerFunc) *contextHandler {
+		return newContextHandler(context, handler)
+	}
+
+	clustersRouter := apiRouter.PathPrefix("/rotate").Subrouter()
+	clustersRouter.Handle("", addContext(handleRotateCluster)).Methods("POST")
+}
+
+// handleRotateCluster responds to POST /api/rotate, beginning the process of rotating a k8s cluster.
+// sample body:
+// {
+//     "clusterID": "12345678",
+//     "maxScaling": 2,
+//     "rotateMasters":  true,
+//     "rotateWorkers": true,
+//     "maxDrainRetries": 10,
+//     "EvictGracePeriod": 60,
+//     "WaitBetweenRotations": 60,
+//     "WaitBetweenDrains": 60,
+// }
+func handleRotateCluster(c *Context, w http.ResponseWriter, r *http.Request) {
+
+	rotateClusterRequest, err := model.NewRotateClusterRequestFromReader(r.Body)
+	if err != nil {
+		c.Logger.WithError(err).Error("failed to decode request")
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	cluster := model.Cluster{
+		ClusterID:            rotateClusterRequest.ClusterID,
+		MaxScaling:           rotateClusterRequest.MaxScaling,
+		RotateMasters:        rotateClusterRequest.RotateMasters,
+		RotateWorkers:        rotateClusterRequest.RotateWorkers,
+		MaxDrainRetries:      rotateClusterRequest.MaxDrainRetries,
+		EvictGracePeriod:     rotateClusterRequest.EvictGracePeriod,
+		WaitBetweenRotations: rotateClusterRequest.WaitBetweenRotations,
+		WaitBetweenDrains:    rotateClusterRequest.WaitBetweenDrains,
+	}
+
+	rotatorMetada := rotator.RotatorMetadata{}
+
+	go rotator.InitRotateCluster(&cluster, &rotatorMetada, c.Logger.WithField("cluster", cluster.ClusterID))
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusAccepted)
+	outputJSON(c, w, cluster)
+}

api/common.go

@@ -0,0 +1,18 @@
+package api
+
+import (
+	"encoding/json"
+	"io"
+)
+
+// outputJSON is a helper method to write the given data as JSON to the given writer.
+//
+// It only logs an error if one occurs, rather than returning, since there is no point in trying
+// to send a new status code back to the client once the body has started sending.
+func outputJSON(c *Context, w io.Writer, data interface{}) {
+	encoder := json.NewEncoder(w)
+	err := encoder.Encode(data)
+	if err != nil {
+		c.Logger.WithError(err).Error("failed to encode result")
+	}
+}

api/context.go

@@ -0,0 +1,18 @@
+package api
+
+import "github.com/sirupsen/logrus"
+
+// Context provides the API with all necessary data and interfaces for responding to requests.
+//
+// It is cloned before each request, allowing per-request changes such as logger annotations.
+type Context struct {
+	RequestID string
+	Logger    logrus.FieldLogger
+}
+
+// Clone creates a shallow copy of context, allowing clones to apply per-request changes.
+func (c *Context) Clone() *Context {
+	return &Context{
+		Logger: c.Logger,
+	}
+}

api/handler.go

@@ -0,0 +1,34 @@
+package api
+
+import (
+	"net/http"
+
+	"github.com/mattermost/rotator/model"
+	log "github.com/sirupsen/logrus"
+)
+
+type contextHandlerFunc func(c *Context, w http.ResponseWriter, r *http.Request)
+
+type contextHandler struct {
+	context *Context
+	handler contextHandlerFunc
+}
+
+// ServeHTTP gets the http Request
+func (h contextHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	context := h.context.Clone()
+	context.RequestID = model.NewID()
+	context.Logger = context.Logger.WithFields(log.Fields{
+		"path":    r.URL.Path,
+		"request": context.RequestID,
+	})
+
+	h.handler(context, w, r)
+}
+
+func newContextHandler(context *Context, handler contextHandlerFunc) *contextHandler {
+	return &contextHandler{
+		context: context,
+		handler: handler,
+	}
+}