Update allocations no longer require that healthy nodes be destroyed (#16)

Currently, adding an additional node to an existing Redis Cluster would change
the flags being passed to the `attache-control` sidecar task for every existing
allocation. The Nomad scheduler would (correctly) trigger a destructive update
(e.g. reallocate, stop, migrate, and restart) of each existing Redis Cluster
node even though they were already healthy. This is because the Nomad scheduler
can only update an allocation in-place when there are no attributes (environment
variables, file templates, etc.) relevant to any of that job's tasks being
updated.

This PR updates `attache-control` to fetch these counts from a Consul KV path
instead.

To ensure consistency between, the scaling configuration stored in the Consul
and the total number of nodes in the Nomad job specification, I've added a
Terraform file that sets both of them. For more information, see the updated
README.

Lastly, we're approaching the point where more folks may begin to touch this
code so I've also taken the time to comment up my exported structs, fields, and
methods.

Author: beautifulentropy

.gitignore

@@ -48,3 +48,7 @@ attache-control
 
 # Mac
 *.DS_Store
+
+# Terraform
+*.terraform*
+*.tfstate*

README.md

@@ -25,7 +25,7 @@ once they've joined a cluster.
 #### Usage
 ```shell
 $ attache-check -help
-Usage of ./attache-check:
+Usage of attache-check:
   -check-serv-addr string
     	address this utility should listen on (e.g. 127.0.0.1:8080)
   -redis-auth-password-file string
@@ -49,11 +49,11 @@ An ephemeral sidecar that acts as an agent for each Redis node when it's
 started. If a node's `node info` reflects that of a new node, this agent will
 attempt to introduce it to an existing Redis Cluster, if it exists, else it will
 attempt to orchestrate the create a new Redis Cluster if there are enough new
-Redis Nodes (in the Await Consul Service) to do so.
+Redis nodes (in the Await Consul Service) to do so.
 
 #### Usage
 ```shell
-$ attache-control -help
+$ ./attache-control -help
 Usage of ./attache-control:
   -attempt-interval duration
     	Duration to wait between attempts to join or create a cluster (default 3s)
@@ -87,10 +87,6 @@ Usage of ./attache-control:
     	Redis username, (required)
   -redis-node-addr string
     	redis-server listening address, (required)
-  -redis-primary-count int
-    	Total number of expected Redis shard primary nodes, (required)
-  -redis-replica-count int
-    	Total number of expected Redis shard replica nodes
   -redis-tls-ca-cert string
     	Redis client CA certificate file, (required)
   -redis-tls-cert-file string
@@ -100,29 +96,52 @@ Usage of ./attache-control:
 ```
 
 ### Running the Example Nomad Job
-Note: these steps assume that you have the `nomad` and `consul` binaries installed
-on your machine and that they exist in your `PATH`.
+Note: these steps assume that you have the `nomad`, `consul`, and `terraform`
+binaries installed on your machine and that they exist in your `PATH`.
 
 Build the attache-control and attache-check binaries:
 ```shell
 $ go build -o attache-check ./cmd/attache-check/main.go && go build -o attache-control ./cmd/attache-control/main.go ./cmd/attache-control/config.go
 ```
 
-Start the Consul server in `dev` mode:
+In another shell, start the Consul server in `dev` mode:
 ```shell
 $ consul agent -dev -datacenter dev-general -log-level ERROR
 ```
 
-Start the Nomad server in `dev` mode:
+In another shell, start the Nomad server in `dev` mode:
 ```shell
 $ sudo nomad agent -dev -bind 0.0.0.0 -log-level ERROR -dc dev-general
 ```
 
-Start a Nomad job deployment:
+Start a Nomad job deployment using Terraform:
 ```shell
-$ nomad job run -verbose -var-file=./example/vars-file.hcl ./example/job-specification.hcl
+cd example
+terraform init
+terraform plan
+terraform apply
 ```
 
-Open the Nomad UI: http://localhost:4646/ui
+Open the Nomad UI: http://localhost:4646/ui to view information about the Redis
+Cluster deployment
 
-Open the Consul UI: http://localhost:8500/ui
+Open the Consul UI: http://localhost:8500/ui to view health check information
+for the Redis Cluster
+
+### Useful Commands
+
+#### Purge Nomad Job
+This is useful for stopping and garbage collecting a job in Nomad immediately.
+```shell
+nomad job stop -purge "<jobname>"
+```
+
+#### Count Primary Nodes
+```shell
+redis-cli -p <tls-port> --tls --cert ./example/tls/redis/cert.pem --key ./example/tls/redis/key.pem --cacert ./example/tls/ca-cert.pem --user replication-user --pass <redis-password> cluster nodes | grep master | wc -l
+```
+
+#### Count Replica Nodes
+```shell
+redis-cli -p <tls-port> --tls --cert ./example/tls/redis/cert.pem --key ./example/tls/redis/key.pem --cacert ./example/tls/ca-cert.pem --user replication-user --pass <redis-password> cluster nodes | grep slave | wc -l
+```

cmd/attache-check/main.go

@@ -15,12 +15,17 @@ import (
 	logger "github.com/sirupsen/logrus"
 )
 
-// CheckHandler is a wraps an inner redis client with some methods for handling
-// health check requests.
+// CheckHandler wraps an inner redis client and provides a method for handling a
+// health check request from Consul. It's exported for use with with a request
+// router.
 type CheckHandler struct {
 	redis.Client
 }
 
+// StateOK handles health checks from Consul. A 200 response from this handler
+// means that, from this Redis Cluster node's perspective, the Redis Cluster
+// State is OK and Consul can begin advertising this node as part of the Redis
+// Cluster in the Service Catalog.
 func (h *CheckHandler) StateOk(w http.ResponseWriter, r *http.Request) {
 	clusterInfo, err := h.GetClusterInfo()
 	if err != nil {
@@ -52,11 +57,29 @@ func main() {
 		logger.Fatal("Missing required opt 'check-serv-addr'")
 	}
 
-	err := redisOpts.Validate()
-	if err != nil {
-		logger.Fatal(err)
+	if redisOpts.NodeAddr == "" {
+		logger.Fatal("missing required opt: 'redis-node-addr'")
+	}
+
+	if redisOpts.Username == "" {
+		logger.Fatal("missing required opt: 'redis-auth-username'")
+	}
+
+	if redisOpts.PasswordFile == "" {
+		logger.Fatal("missing required opt: 'redis-auth-password-file'")
 	}
 
+	if redisOpts.CACertFile == "" {
+		logger.Fatal("missing required opt: 'redis-tls-ca-cert'")
+	}
+
+	if redisOpts.CertFile == "" {
+		logger.Fatal("missing required opt: 'redis-tls-cert-file'")
+	}
+
+	if redisOpts.KeyFile == "" {
+		logger.Fatal("missing required opt: 'redis-tls-key-file'")
+	}
 	logger.Infof("starting %s", os.Args[0])
 
 	router := mux.NewRouter()

cmd/attache-control/config.go

@@ -9,57 +9,109 @@ import (
 	r "github.com/letsencrypt/attache/src/redis/config"
 )
 
-// CLIOpts is exported for use with flag.Parse().
-type CLIOpts struct {
-	RedisOpts         r.RedisOpts
-	RedisPrimaryCount int
-	RedisReplicaCount int
-	LockPath          string
-	AttemptInterval   time.Duration
-	AttemptLimit      int
-	AwaitServiceName  string
-	DestServiceName   string
-	LogLevel          string
-	ConsulOpts        c.ConsulOpts
-}
+// cliOpts contains all of the configuration used to orchestrate the Redis
+// Cluster under management by Attaché.
+type cliOpts struct {
+	// lockPath is the Consul KV path to use as a leader lock for Redis Cluster
+	// operations.
+	lockPath string
 
-func (c CLIOpts) Validate() error {
-	if c.RedisPrimaryCount == 0 {
-		return errors.New("missing required opt: 'redis-primary-count'")
-	}
+	// attemptInterval is duration to wait between attempts to join or create a
+	// cluster.
+	attemptInterval time.Duration
+
+	// attemptLimit is the number of times to attempt joining or creating a cluster before Attache
+	// should exit as failed.
+	attemptLimit int
+
+	// awaitServiceName is the name of the Consul Service that newly created
+	// Redis Cluster nodes will join when they're first started but have yet to
+	// form or join a cluster. This field is required.
+	awaitServiceName string
+
+	// destServiceName is the name of the Consul Service that Redis Cluster
+	// nodes will join once they are part of a cluster. This field is required.
+	destServiceName string
+
+	// logLevel is the level that Attaché should log at.
+	logLevel string
 
-	if c.DestServiceName == "" {
+	// RedisOpts contains the configuration for interacting with the node this
+	// serves as a sidecar to and, if one exists, the Redis Cluster. This field
+	// is required.
+	RedisOpts r.RedisOpts
+
+	// ConsulOpts contains the configuration for interacting with the Consul
+	// cluster that Attaché uses for leader lock and to retrieve the scaling
+	// options in the Consul KV store. This field is required.
+	ConsulOpts c.ConsulOpts
+}
+
+// Validate checks that the required opts for `attache-control` were passed via
+// the CLI. User friendly errors are returned when this is not the case.
+func (c cliOpts) Validate() error {
+	if c.destServiceName == "" {
 		return errors.New("missing required opt: 'dest-service-name'")
 	}
 
-	if c.AwaitServiceName == "" {
+	if c.awaitServiceName == "" {
 		return errors.New("missing required opt: 'await-service-name'")
 	}
 
-	err := c.ConsulOpts.Validate()
-	if err != nil {
-		return err
+	if c.ConsulOpts.EnableTLS {
+		if c.ConsulOpts.TLSCACertFile == "" {
+			return errors.New("missing required opt: 'consul-tls-ca-cert")
+		}
+
+		if c.ConsulOpts.TLSCertFile == "" {
+			return errors.New("missing required opt: 'consul-tls-cert")
+		}
+
+		if c.ConsulOpts.TLSKeyFile == "" {
+			return errors.New("missing required opt: 'consul-tls-key")
+		}
+	}
+
+	if !c.ConsulOpts.EnableTLS && (c.ConsulOpts.TLSCACertFile != "" || c.ConsulOpts.TLSCertFile != "" || c.ConsulOpts.TLSKeyFile != "") {
+		return errors.New("missing required opt: 'consul-tls-enable")
+	}
+
+	if c.RedisOpts.NodeAddr == "" {
+		return errors.New("missing required opt: 'redis-node-addr'")
+	}
+
+	if c.RedisOpts.Username == "" {
+		return errors.New("missing required opt: 'redis-auth-username'")
+	}
+
+	if c.RedisOpts.PasswordFile == "" {
+		return errors.New("missing required opt: 'redis-auth-password-file'")
+	}
+
+	if c.RedisOpts.CACertFile == "" {
+		return errors.New("missing required opt: 'redis-tls-ca-cert'")
+	}
+
+	if c.RedisOpts.CertFile == "" {
+		return errors.New("missing required opt: 'redis-tls-cert-file'")
 	}
 
-	err = c.RedisOpts.Validate()
-	if err != nil {
-		return err
+	if c.RedisOpts.KeyFile == "" {
+		return errors.New("missing required opt: 'redis-tls-key-file'")
 	}
 	return nil
 }
 
-func ParseFlags() CLIOpts {
-	var conf CLIOpts
+func ParseFlags() cliOpts {
+	var conf cliOpts
 
 	// CLI
-	flag.IntVar(&conf.RedisPrimaryCount, "redis-primary-count", 0, "Total number of expected Redis shard primary nodes, (required)")
-	flag.IntVar(&conf.RedisReplicaCount, "redis-replica-count", 0, "Total number of expected Redis shard replica nodes")
-	flag.StringVar(&conf.LockPath, "lock-kv-path", "service/attache/leader", "Consul KV path used as a distributed lock for operations")
-	flag.DurationVar(&conf.AttemptInterval, "attempt-interval", 3*time.Second, "Duration to wait between attempts to join or create a cluster")
-	flag.IntVar(&conf.AttemptLimit, "attempt-limit", 20, "Number of times to attempt joining or creating a cluster before exiting")
-	flag.StringVar(&conf.AwaitServiceName, "await-service-name", "", "Consul Service for newly created Redis Cluster Nodes, (required)")
-	flag.StringVar(&conf.DestServiceName, "dest-service-name", "", "Consul Service for healthy Redis Cluster Nodes, (required)")
-	flag.StringVar(&conf.LogLevel, "log-level", "info", "Set the log level")
+	flag.StringVar(&conf.lockPath, "lock-kv-path", "service/attache/leader", "Consul KV path to use as a leader lock for Redis Cluster operations")
+	flag.DurationVar(&conf.attemptInterval, "attempt-interval", 3*time.Second, "Duration to wait between attempts to join or create a cluster (e.g. '1s')")
+	flag.IntVar(&conf.attemptLimit, "attempt-limit", 20, "Number of times to attempt for or join a cluster before exiting")
+	flag.StringVar(&conf.awaitServiceName, "await-service-name", "", "Consul Service for newly created Redis Cluster Nodes, (required)")
+	flag.StringVar(&conf.destServiceName, "dest-service-name", "", "Consul Service for healthy Redis Cluster Nodes, (required)")
+	flag.StringVar(&conf.logLevel, "log-level", "info", "Set the log level")
 
 	// Redis
 	flag.StringVar(&conf.RedisOpts.NodeAddr, "redis-node-addr", "", "redis-server listening address, (required)")
@@ -73,10 +125,10 @@ func ParseFlags() CLIOpts {
 	flag.StringVar(&conf.ConsulOpts.DC, "consul-dc", "dev-general", "Consul client datacenter")
 	flag.StringVar(&conf.ConsulOpts.Address, "consul-addr", "127.0.0.1:8500", "Consul client address")
 	flag.StringVar(&conf.ConsulOpts.ACLToken, "consul-acl-token", "", "Consul client ACL token")
-	flag.BoolVar(&conf.ConsulOpts.EnableTLS, "consul-tls-enable", false, "Enable mTLS for the Consul client")
-	flag.StringVar(&conf.ConsulOpts.TLSCACert, "consul-tls-ca-cert", "", "Consul client CA certificate file")
-	flag.StringVar(&conf.ConsulOpts.TLSCert, "consul-tls-cert", "", "Consul client certificate file")
-	flag.StringVar(&conf.ConsulOpts.TLSKey, "consul-tls-key", "", "Consul client key file")
+	flag.BoolVar(&conf.ConsulOpts.EnableTLS, "consul-tls-enable", false, "Enable mTLS for the Consul client (requires 'consul-tls-ca-cert', 'consul-tls-cert', 'consul-tls-key')")
+	flag.StringVar(&conf.ConsulOpts.TLSCACertFile, "consul-tls-ca-cert", "", "Consul client CA certificate file")
+	flag.StringVar(&conf.ConsulOpts.TLSCertFile, "consul-tls-cert", "", "Consul client certificate file")
+	flag.StringVar(&conf.ConsulOpts.TLSKeyFile, "consul-tls-key", "", "Consul client key file")
 
 	flag.Parse()
 	return conf