From 3e00b8e44a602cbcc0f5e9e6c9fe43685f98741b Mon Sep 17 00:00:00 2001 From: jean-claude cote Date: Fri, 1 Sep 2017 12:17:52 -0400 Subject: [PATCH 1/8] Added support for Swarm tasks running on worker nodes --- config/config.go | 1 + .../nginx-swarm-stack-machine/README.md | 66 +++++ .../docker-compose.yml | 58 +++++ .../nginx-template.conf | 156 ++++++++++++ ext/lb/haproxy/generate.go | 196 ++++++++++++++- ext/lb/lb.go | 92 +++++-- ext/lb/nginx/generate.go | 234 +++++++++++++++++- ext/lb/utils/alias_domains.go | 5 +- ext/lb/utils/backend_options.go | 5 +- ext/lb/utils/balance.go | 5 +- ext/lb/utils/context_root.go | 9 +- ext/lb/utils/domain.go | 5 +- ext/lb/utils/health_check.go | 9 +- ext/lb/utils/hostname.go | 5 +- ext/lb/utils/ip_hash.go | 5 +- ext/lb/utils/network.go | 19 +- ext/lb/utils/ssl.go | 25 +- ext/lb/utils/websocket.go | 5 +- 18 files changed, 801 insertions(+), 99 deletions(-) create mode 100644 docs/examples/nginx-swarm-stack-machine/README.md create mode 100644 docs/examples/nginx-swarm-stack-machine/docker-compose.yml create mode 100644 docs/examples/nginx-swarm-stack-machine/nginx-template.conf diff --git a/config/config.go b/config/config.go index 4acc35f1..9902a800 100644 --- a/config/config.go +++ b/config/config.go @@ -22,6 +22,7 @@ type ExtensionConfig struct { MaxConn int // haproxy, nginx Port int // haproxy, nginx SyslogAddr string // haproxy + SwarmTaskMode bool // haproxy, nginx AdminUser string // haproxy AdminPass string // haproxy SSLCertPath string // haproxy, nginx diff --git a/docs/examples/nginx-swarm-stack-machine/README.md b/docs/examples/nginx-swarm-stack-machine/README.md new file mode 100644 index 00000000..57b634ca --- /dev/null +++ b/docs/examples/nginx-swarm-stack-machine/README.md @@ -0,0 +1,66 @@ +# Interlock + Docker Swarm + Stack deploy +This example shows a Interlock in a Swarm cluster deployed using docker stack deploy. + +Start with the [Docker Swarm](https://docs.docker.com/swarm/install-w-machine/) +evaluation tutorial. Once you have a working Swarm cluster continue below. + +Note: you need a manager and a worker node to run this example + +Note: this uses [Docker Compose](http://docs.docker.com/compose). Please make +sure you have the latest version installed. + +# Setup +To make this example portable, we use an environment variable to configure +Interlock to your Swarm cluster. Run the following to set it up: + +`docker-machine env manager` + +export DOCKER_TLS_VERIFY="1" +export DOCKER_HOST="tcp://192.168.99.102:2376" +export DOCKER_CERT_PATH="/Users/jccote/.docker/machine/machines/manager" +export DOCKER_MACHINE_NAME="manager" +# Run this command to configure your shell: +# eval $(docker-machine env manager) + +# generate a stack file using docker-compose +`docker-compose -f ./docs/examples/nginx-swarm-stack-machine/docker-compose.yml config > stack.yml` + +# deploy the stack using docker stack deploy and give your stack a name +`docker stack deploy -c stack.yml mystack` + +# you should now have the following service running +`docker service ls` +ID NAME MODE REPLICAS IMAGE PORTS +6jbqsojcwrbb mystack_app replicated 2/2 ehazlett/docker-demo:latest *:0->8080/tcp +kbeckpeyqbob mystack_nginx replicated 1/1 nginx:latest *:80->80/tcp +ykdsht0davud mystack_interlock replicated 1/1 ehazlett/interlock:jcc + +Once up you can check the logs to ensure Interlock is detecting: + +`docker logs mystack_interlock` + + +You can also verify that the nginx routes are created properly: +`docker exec -it mystack_nginx.1.d2tt5tdwcsz0yq91wjwhympqy /bin/bash -c "cat /etc/nginx/nginx.conf"` + + upstream ctx___web { + zone ctx___web_backend 64k; + server 10.0.0.8:8080; + server 10.0.0.5:8080; + + } + + server { + listen 7070; + server_name _; + + + location /web { + + proxy_pass http://ctx___web; + } + + +The sample web applications should be available at +http://192.168.99.102/web + diff --git a/docs/examples/nginx-swarm-stack-machine/docker-compose.yml b/docs/examples/nginx-swarm-stack-machine/docker-compose.yml new file mode 100644 index 00000000..2bb37973 --- /dev/null +++ b/docs/examples/nginx-swarm-stack-machine/docker-compose.yml @@ -0,0 +1,58 @@ +version: '3' + +networks: + overlay_net: + +services: + + interlock: + image: ehazlett/interlock:jcmcote + command: -D run -c /etc/interlock/config.toml + tty: true + networks: + - overlay_net + deploy: + replicas: 1 + placement: + constraints: + [ node.role == manager ] + volumes: + - ${PWD}/docs/examples/nginx-swarm-stack-machine/config.toml:/etc/interlock/config.toml:ro + - ${PWD}/docs/examples/nginx-swarm-stack-machine/nginx-template.conf:/nginx-template.conf:ro + - ${DOCKER_CERT_PATH}:/var/lib/boot2docker:ro + + + nginx: + image: nginx:latest + entrypoint: nginx + command: -g "daemon off;" -c /etc/nginx/nginx.conf + networks: + - overlay_net + deploy: + replicas: 1 + placement: + constraints: + [ node.role == manager ] + ports: + - 80:80 + - 7070:7070 + labels: + - "interlock.ext.name=nginx" + + app: + image: ehazlett/docker-demo:latest + networks: + - overlay_net + deploy: + replicas: 2 + placement: + constraints: + [ node.role != manager ] + ports: + - 8080:8080 + labels: + - "interlock.hostname=" + - "interlock.domain=_" + - "interlock.context_root=/web" + - "interlock.port=8080" + - "interlock.network=mystack_overlay_net" diff --git a/docs/examples/nginx-swarm-stack-machine/nginx-template.conf b/docs/examples/nginx-swarm-stack-machine/nginx-template.conf new file mode 100644 index 00000000..e3df08ce --- /dev/null +++ b/docs/examples/nginx-swarm-stack-machine/nginx-template.conf @@ -0,0 +1,156 @@ +user {{ .Config.User }}; +worker_processes {{ .Config.WorkerProcesses }}; +worker_rlimit_nofile {{ .Config.RLimitNoFile }}; + +error_log /var/log/error.log warn; +pid {{ .Config.PidPath }}; + + +events { + worker_connections {{ .Config.MaxConn }}; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + server_names_hash_bucket_size 128; + client_max_body_size 2048M; + + log_format main '$remote_addr - $remote_user [$upstream_addr] [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + # If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the + # scheme used to connect to this server + map $http_x_forwarded_proto $proxy_x_forwarded_proto { + default $http_x_forwarded_proto; + '' $scheme; + } + + #gzip on; + proxy_connect_timeout {{ .Config.ProxyConnectTimeout }}; + proxy_send_timeout {{ .Config.ProxySendTimeout }}; + proxy_read_timeout {{ .Config.ProxyReadTimeout }}; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; + proxy_set_header Host $http_host; + send_timeout {{ .Config.SendTimeout }}; + + # ssl + ssl_prefer_server_ciphers on; + ssl_ciphers {{ .Config.SSLCiphers }}; + ssl_protocols {{ .Config.SSLProtocols }}; + {{ if .Config.DHParam}}ssl_dhparam {{ .Config.DHParamPath }};{{ end }} + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + # default host return 503 + server { + listen {{ 80 }}; + server_name _; + + location / { + return 503; + } + + location /nginx_status { + stub_status on; + access_log off; + } + } + + {{ range $host := .Hosts }} + {{ if $host.Upstream.Servers }} + upstream {{ $host.Upstream.Name }} { + {{ if $host.IPHash }}ip_hash; {{else}}zone {{ $host.Upstream.Name }}_backend 64k;{{ end }} + + {{ range $up := $host.Upstream.Servers }}server {{ $up.Addr }}; + {{ end }} + } + {{ end }} + {{ range $k, $ctxroot := $host.ContextRoots }} + upstream ctx{{ $k }} { + {{ if $host.IPHash }}ip_hash; {{else}}zone ctx{{ $ctxroot.Name }}_backend 64k;{{ end }} + {{ range $d := $ctxroot.Upstreams }}server {{ $d }}; + {{ end }} + } {{ end }} + + server { + listen {{ $host.Port }}; + server_name{{ range $name := $host.ServerNames }} {{ $name }}{{ end }}; + + {{ range $ctxroot := $host.ContextRoots }} + location {{ $ctxroot.Path }} { + {{ if $ctxroot.Rewrite }}rewrite ^([^.]*[^/])$ $1/ permanent; + rewrite ^{{ $ctxroot.Path }}/(.*) /$1 break;{{ end }} + proxy_pass http://ctx{{ $ctxroot.Name }}; + } + {{ end }} + + {{ if $host.SSLOnly }}return 302 https://$server_name$request_uri;{{ else }} + {{ if $host.Upstream.Servers }} + location / { + {{ if $host.SSLBackend }}proxy_pass https://{{ $host.Upstream.Name }};{{ else }}proxy_pass http://{{ $host.Upstream.Name }};{{ end }} + } + {{ end }} + + {{ range $ws := $host.WebsocketEndpoints }} + location {{ $ws }} { + {{ if $host.SSLBackend }}proxy_pass https://{{ $host.Upstream.Name }};{{ else }}proxy_pass http://{{ $host.Upstream.Name }};{{ end }} + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } + + location /nginx_status { + stub_status on; + access_log off; + } + + {{ end }} + } + {{ if $host.SSL }} + server { + listen {{ $host.SSLPort }}; + ssl on; + ssl_certificate {{ $host.SSLCert }}; + ssl_certificate_key {{ $host.SSLCertKey }}; + server_name{{ range $name := $host.ServerNames }} {{ $name }}{{ end }}; + + location / { + {{ if $host.SSLBackend }}proxy_pass https://{{ $host.Upstream.Name }};{{ else }}proxy_pass http://{{ $host.Upstream.Name }};{{ end }} + } + + {{ range $ws := $host.WebsocketEndpoints }} + location {{ $ws }} { + {{ if $host.SSLBackend }}proxy_pass https://{{ $host.Upstream.Name }};{{ else }}proxy_pass http://{{ $host.Upstream.Name }};{{ end }} + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } + + location /nginx_status { + stub_status on; + access_log off; + } + {{ end }} + } + {{ end }} + + {{ end }} + {{ end }} {{/* end host range */}} + + include {{ .Config.ConfigBasePath }}/conf.d/*.conf; +} diff --git a/ext/lb/haproxy/generate.go b/ext/lb/haproxy/generate.go index 894e112f..a52a3c56 100644 --- a/ext/lb/haproxy/generate.go +++ b/ext/lb/haproxy/generate.go @@ -7,6 +7,8 @@ import ( "github.com/docker/docker/api/types" "github.com/ehazlett/interlock/ext/lb/utils" "golang.org/x/net/context" + "github.com/docker/docker/api/types/swarm" + "net" ) func (p *HAProxyLoadBalancer) GenerateProxyConfig(containers []types.Container) (interface{}, error) { @@ -26,12 +28,13 @@ func (p *HAProxyLoadBalancer) GenerateProxyConfig(containers []types.Container) for _, c := range containers { cntId := c.ID[:12] + labels := c.Labels // load interlock data - hostname := utils.Hostname(c) - domain := utils.Domain(c) + hostname := utils.Hostname(labels) + domain := utils.Domain(labels) // context root - contextRoot := utils.ContextRoot(c) + contextRoot := utils.ContextRoot(labels) contextRootName := strings.Replace(contextRoot, "/", "_", -1) if domain == "" && contextRoot == "" { @@ -46,10 +49,10 @@ func (p *HAProxyLoadBalancer) GenerateProxyConfig(containers []types.Container) Name: contextRootName, Path: contextRoot, } - hostContextRootRewrites[domain] = utils.ContextRootRewrite(c) + hostContextRootRewrites[domain] = utils.ContextRootRewrite(labels) - healthCheck := utils.HealthCheck(c) - healthCheckInterval, err := utils.HealthCheckInterval(c) + healthCheck := utils.HealthCheck(labels) + healthCheckInterval, err := utils.HealthCheckInterval(labels) if err != nil { log().Errorf("error parsing health check interval: %s", err) continue @@ -69,25 +72,25 @@ func (p *HAProxyLoadBalancer) GenerateProxyConfig(containers []types.Container) log().Debugf("check interval for %s: %d", domain, healthCheckInterval) } - hostBalanceAlgorithms[domain] = utils.BalanceAlgorithm(c) + hostBalanceAlgorithms[domain] = utils.BalanceAlgorithm(labels) - backendOptions := utils.BackendOptions(c) + backendOptions := utils.BackendOptions(labels) if len(backendOptions) > 0 { hostBackendOptions[domain] = backendOptions log().Debugf("using backend options for %s: %s", domain, strings.Join(backendOptions, ",")) } - hostSSLOnly[domain] = utils.SSLOnly(c) + hostSSLOnly[domain] = utils.SSLOnly(labels) // ssl backend - hostSSLBackend[domain] = utils.SSLBackend(c) - hostSSLBackendTLSVerify[domain] = utils.SSLBackendTLSVerify(c) + hostSSLBackend[domain] = utils.SSLBackend(labels) + hostSSLBackendTLSVerify[domain] = utils.SSLBackendTLSVerify(labels) addr := "" // check for networking - if n, ok := utils.OverlayEnabled(c); ok { + if n, ok := utils.OverlayEnabled(labels); ok { log().Debugf("configuring docker network: name=%s", n) network, err := p.client.NetworkInspect(context.Background(), n, false) @@ -127,7 +130,174 @@ func (p *HAProxyLoadBalancer) GenerateProxyConfig(containers []types.Container) log().Infof("%s: upstream=%s container=%s", domain, addr, container_name) // "parse" multiple labels for alias domains - aliasDomains := utils.AliasDomains(c) + aliasDomains := utils.AliasDomains(labels) + + log().Debugf("alias domains: %v", aliasDomains) + + for _, alias := range aliasDomains { + log().Debugf("adding alias %s for %s", alias, cntId) + proxyUpstreams[alias] = append(proxyUpstreams[alias], up) + hostContextRoots[alias] = &ContextRoot{ + Name: contextRootName, + Path: contextRoot, + } + } + + proxyUpstreams[domain] = append(proxyUpstreams[domain], up) + } + + for k, v := range proxyUpstreams { + name := strings.Replace(k, ".", "_", -1) + host := &Host{ + Name: name, + ContextRoot: hostContextRoots[k], + ContextRootRewrite: hostContextRootRewrites[k], + Domain: k, + Upstreams: v, + Check: hostChecks[k], + BalanceAlgorithm: hostBalanceAlgorithms[k], + BackendOptions: hostBackendOptions[k], + SSLOnly: hostSSLOnly[k], + SSLBackend: hostSSLBackend[k], + SSLBackendTLSVerify: hostSSLBackendTLSVerify[k], + } + log().Debugf("adding host name=%s domain=%s contextroot=%v", host.Name, host.Domain, host.ContextRoot) + hosts = append(hosts, host) + } + + cfg := &Config{ + Hosts: hosts, + Config: p.cfg, + Networks: networks, + } + + return cfg, nil +} + +func (p *HAProxyLoadBalancer) GenerateProxyConfigForTasks(tasks []swarm.Task) (interface{}, error) { + var hosts []*Host + + proxyUpstreams := map[string][]*Upstream{} + hostChecks := map[string]string{} + hostBalanceAlgorithms := map[string]string{} + hostContextRoots := map[string]*ContextRoot{} + hostContextRootRewrites := map[string]bool{} + hostBackendOptions := map[string][]string{} + hostSSLOnly := map[string]bool{} + hostSSLBackend := map[string]bool{} + hostSSLBackendTLSVerify := map[string]string{} + + networks := map[string]string{} + + for _, t := range tasks { + cntId := t.ID[:12] + labels := t.Labels + // load interlock data + hostname := utils.Hostname(labels) + domain := utils.Domain(labels) + + // context root + contextRoot := utils.ContextRoot(labels) + contextRootName := strings.Replace(contextRoot, "/", "_", -1) + + if domain == "" && contextRoot == "" { + continue + } + + if hostname != domain && hostname != "" { + domain = fmt.Sprintf("%s.%s", hostname, domain) + } + + hostContextRoots[domain] = &ContextRoot{ + Name: contextRootName, + Path: contextRoot, + } + hostContextRootRewrites[domain] = utils.ContextRootRewrite(labels) + + healthCheck := utils.HealthCheck(labels) + healthCheckInterval, err := utils.HealthCheckInterval(labels) + if err != nil { + log().Errorf("error parsing health check interval: %s", err) + continue + } + + if healthCheck != "" { + if val, ok := hostChecks[domain]; ok { + // check existing host check for different values + if val != healthCheck { + log().Warnf("conflicting check specified for %s", domain) + } + } else { + hostChecks[domain] = healthCheck + log().Debugf("using custom check for %s: %s", domain, healthCheck) + } + + log().Debugf("check interval for %s: %d", domain, healthCheckInterval) + } + + hostBalanceAlgorithms[domain] = utils.BalanceAlgorithm(labels) + + backendOptions := utils.BackendOptions(labels) + + if len(backendOptions) > 0 { + hostBackendOptions[domain] = backendOptions + log().Debugf("using backend options for %s: %s", domain, strings.Join(backendOptions, ",")) + } + + hostSSLOnly[domain] = utils.SSLOnly(labels) + + // ssl backend + hostSSLBackend[domain] = utils.SSLBackend(labels) + hostSSLBackendTLSVerify[domain] = utils.SSLBackendTLSVerify(labels) + + addr := "" + + interlockPort, err := utils.CustomPort(labels) + if err != nil { + log().Error(err) + continue + } + log().Debug(interlockPort) + + // check for networking + if overlayNetworkName, ok := utils.OverlayEnabled(labels); ok { + log().Debugf("configuring docker network: name=%s", overlayNetworkName) + + for _, networksAttachment := range t.NetworksAttachments { + + if overlayNetworkName == networksAttachment.Network.Spec.Annotations.Name { + for _, address := range networksAttachment.Addresses { + log().Debug(address) + + ip, _, err := net.ParseCIDR(address) + if err != nil { + log().Error(err) + continue + } + + addr = fmt.Sprintf("%s:%d", ip, interlockPort) + log().Debug(addr) + } + } + } + + networks[overlayNetworkName] = "" + } else { + + //addr = fmt.Sprintf("%s:%d", network, interlockPort) + } + + container_name := t.ID + up := &Upstream{ + Addr: addr, + Container: container_name, + CheckInterval: healthCheckInterval, + } + + log().Infof("%s: upstream=%s container=%s", domain, addr, container_name) + + // "parse" multiple labels for alias domains + aliasDomains := utils.AliasDomains(labels) log().Debugf("alias domains: %v", aliasDomains) diff --git a/ext/lb/lb.go b/ext/lb/lb.go index d95ec6f7..e4c1ae0d 100644 --- a/ext/lb/lb.go +++ b/ext/lb/lb.go @@ -14,17 +14,18 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/api/types/filters" ntypes "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/ehazlett/interlock/config" "github.com/ehazlett/interlock/events" "github.com/ehazlett/interlock/ext" - "github.com/ehazlett/interlock/ext/lb/haproxy" "github.com/ehazlett/interlock/ext/lb/nginx" - "github.com/ehazlett/interlock/utils" + "github.com/ehazlett/interlock/ext/lb/utils" "github.com/ehazlett/ttlcache" "golang.org/x/net/context" + "github.com/ehazlett/interlock/ext/lb/haproxy" ) const ( @@ -48,6 +49,7 @@ type LoadBalancerBackend interface { Name() string ConfigPath() string GenerateProxyConfig(c []types.Container) (interface{}, error) + GenerateProxyConfigForTasks(s []swarm.Task) (interface{}, error) Template() string Reload(proxyContainers []types.Container) error } @@ -110,11 +112,12 @@ func NewLoadBalancer(c *config.ExtensionConfig, client *client.Client) (*LoadBal }) // load containerID for the following nodeID - containerID, err := utils.GetContainerID() - if err != nil { - return nil, err - } +// containerID, err := utils.GetContainerID() +// if err != nil { +// return nil, err +// } + containerID := "none" log().Infof("interlock node: container id=%s", containerID) extension := &LoadBalancer{ @@ -202,27 +205,64 @@ func NewLoadBalancer(c *config.ExtensionConfig, client *client.Client) (*LoadBal log().Debug("updating load balancers") - optFilters := filters.NewArgs() - optFilters.Add("status", "running") - optFilters.Add("label", "interlock.hostname") - opts := types.ContainerListOptions{ - All: false, - Size: false, - Filters: optFilters, - } - log().Debug("getting container list") - containers, err := client.ContainerList(context.Background(), opts) - if err != nil { - errChan <- err - continue - } + var cfg interface{} - // generate proxy config - log().Debug("generating proxy config") - cfg, err := extension.backend.GenerateProxyConfig(containers) - if err != nil { - errChan <- err - continue + if extension.cfg.SwarmTaskMode == true { + optFilters := filters.NewArgs() + // jcc, this filter is not working... + // optFilters.Add("label", "interlock.hostname") + optFilters.Add("desired-state", "running") + opts := types.TaskListOptions{ + Filters: optFilters, + } + log().Debug("getting task list") + tasks, err := client.TaskList(context.Background(), opts) + if err != nil { + errChan <- err + continue + } + + var proxyTasks []swarm.Task + for _, t := range tasks { + labels := t.Spec.ContainerSpec.Labels + hostname := utils.Hostname(labels) + if hostname != "unknown" { + proxyTasks = append(proxyTasks, t) + } + } + // generate proxy config + log().Debug("generating proxy config for tasks") + cfg, err = extension.backend.GenerateProxyConfigForTasks(proxyTasks) + if err != nil { + errChan <- err + continue + } + log().Debugf("service gen conf: %s", cfg) + } else { + optFilters := filters.NewArgs() + optFilters.Add("status", "running") + optFilters.Add("label", "interlock.hostname") + opts := types.ContainerListOptions{ + All: false, + Size: false, + Filters: optFilters, + } + log().Debug("getting container list") + containers, err := client.ContainerList(context.Background(), opts) + if err != nil { + errChan <- err + continue + } + + // generate proxy config + log().Debug("generating proxy config") + cfg, err = extension.backend.GenerateProxyConfig(containers) + if err != nil { + errChan <- err + continue + } + + log().Debugf("container gen conf: %s", cfg) } // save proxy config diff --git a/ext/lb/nginx/generate.go b/ext/lb/nginx/generate.go index 8f11c14a..c18a61e8 100644 --- a/ext/lb/nginx/generate.go +++ b/ext/lb/nginx/generate.go @@ -6,8 +6,10 @@ import ( "strings" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm" "github.com/ehazlett/interlock/ext/lb/utils" "golang.org/x/net/context" + "net" ) func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (interface{}, error) { @@ -27,11 +29,13 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i for _, c := range containers { cntId := c.ID[:12] + labels := c.Labels // load interlock data - contextRoot := utils.ContextRoot(c) + contextRoot := utils.ContextRoot(labels) + + hostname := utils.Hostname(labels) + domain := utils.Domain(labels) - hostname := utils.Hostname(c) - domain := utils.Domain(c) if domain == "" && contextRoot == "" { continue @@ -43,7 +47,7 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i // context root contextRootName := fmt.Sprintf("%s_%s", domain, strings.Replace(contextRoot, "/", "_", -1)) - contextRootRewrite := utils.ContextRootRewrite(c) + contextRootRewrite := utils.ContextRootRewrite(labels) // check if the first server name is there; if not, add // this happens if there are multiple backend containers @@ -51,16 +55,16 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i serverNames[domain] = []string{domain} } - hostSSL[domain] = utils.SSLEnabled(c) - hostSSLOnly[domain] = utils.SSLOnly(c) - hostIPHash[domain] = utils.IPHash(c) + hostSSL[domain] = utils.SSLEnabled(labels) + hostSSLOnly[domain] = utils.SSLOnly(labels) + hostIPHash[domain] = utils.IPHash(labels) // check ssl backend - hostSSLBackend[domain] = utils.SSLBackend(c) + hostSSLBackend[domain] = utils.SSLBackend(labels) // set cert paths baseCertPath := p.cfg.SSLCertPath - certName := utils.SSLCertName(c) + certName := utils.SSLCertName(labels) if certName != "" { certPath := filepath.Join(baseCertPath, certName) @@ -68,7 +72,7 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i hostSSLCert[domain] = certPath } - certKeyName := utils.SSLCertKey(c) + certKeyName := utils.SSLCertKey(labels) if certKeyName != "" { keyPath := filepath.Join(baseCertPath, certKeyName) log().Infof("ssl key for %s: %s", domain, keyPath) @@ -78,7 +82,7 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i addr := "" // check for networking - if n, ok := utils.OverlayEnabled(c); ok { + if n, ok := utils.OverlayEnabled(labels); ok { log().Debugf("configuring docker network: name=%s", n) network, err := p.client.NetworkInspect(context.Background(), n, false) @@ -129,7 +133,7 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i } // "parse" multiple labels for websocket endpoints - websocketEndpoints := utils.WebsocketEndpoints(c) + websocketEndpoints := utils.WebsocketEndpoints(labels) log().Debugf("websocket endpoints: %v", websocketEndpoints) @@ -139,7 +143,7 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i } // "parse" multiple labels for alias domains - aliasDomains := utils.AliasDomains(c) + aliasDomains := utils.AliasDomains(labels) log().Debugf("alias domains: %v", aliasDomains) @@ -200,3 +204,207 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i return config, nil } + +// JCC +func (p *NginxLoadBalancer) GenerateProxyConfigForTasks(tasks []swarm.Task) (interface{}, error) { + var hosts []*Host + upstreamHosts := map[string]struct{}{} + upstreamServers := map[string][]string{} + serverNames := map[string][]string{} + hostContextRoots := map[string]map[string]*ContextRoot{} + hostSSL := map[string]bool{} + hostSSLCert := map[string]string{} + hostSSLCertKey := map[string]string{} + hostSSLOnly := map[string]bool{} + hostSSLBackend := map[string]bool{} + hostWebsocketEndpoints := map[string][]string{} + hostIPHash := map[string]bool{} + networks := map[string]string{} + + // JCC instead of containers I'll have service specifications here + + for _, t := range tasks { + srvId := t.ID[:12] + labels := t.Spec.ContainerSpec.Labels + // load interlock data + contextRoot := utils.ContextRoot(labels) + + hostname := utils.Hostname(labels) + domain := utils.Domain(labels) + + if t.Status.State != swarm.TaskStateRunning { + continue + } + + if domain == "" && contextRoot == "" { + continue + } + + if hostname != domain && hostname != "" { + domain = fmt.Sprintf("%s.%s", hostname, domain) + } + + // context root + contextRootName := fmt.Sprintf("%s_%s", domain, strings.Replace(contextRoot, "/", "_", -1)) + contextRootRewrite := utils.ContextRootRewrite(labels) + + // check if the first server name is there; if not, add + // this happens if there are multiple backend containers + if _, ok := serverNames[domain]; !ok { + serverNames[domain] = []string{domain} + } + + // JCC many of the checks here are for labels on containers, the labels are on the service definition as well. + + hostSSL[domain] = utils.SSLEnabled(labels) + hostSSLOnly[domain] = utils.SSLOnly(labels) + hostIPHash[domain] = utils.IPHash(labels) + // check ssl backend + hostSSLBackend[domain] = utils.SSLBackend(labels) + + // set cert paths + baseCertPath := p.cfg.SSLCertPath + + certName := utils.SSLCertName(labels) + + if certName != "" { + certPath := filepath.Join(baseCertPath, certName) + log().Infof("ssl cert for %s: %s", domain, certPath) + hostSSLCert[domain] = certPath + } + + certKeyName := utils.SSLCertKey(labels) + if certKeyName != "" { + keyPath := filepath.Join(baseCertPath, certKeyName) + log().Infof("ssl key for %s: %s", domain, keyPath) + hostSSLCertKey[domain] = keyPath + } + + addr := "" + + interlockPort, err := utils.CustomPort(labels) + if err != nil { + log().Error(err) + continue + } + log().Debug(interlockPort) + + // check for networking + if overlayNetworkName, ok := utils.OverlayEnabled(labels); ok { + log().Debugf("configuring docker network: name=%s", overlayNetworkName) + + for _, networksAttachment := range t.NetworksAttachments { + + if overlayNetworkName == networksAttachment.Network.Spec.Annotations.Name { + for _, address := range networksAttachment.Addresses { + log().Debug(address) + + ip, _, err := net.ParseCIDR(address) + if err != nil { + log().Error(err) + continue + } + + addr = fmt.Sprintf("%s:%d", ip, interlockPort) + log().Debug(addr) + } + } + } + + networks[overlayNetworkName] = "" + } else { + + //addr = fmt.Sprintf("%s:%d", network, interlockPort) + } + + if contextRoot != "" { + if _, ok := hostContextRoots[domain]; !ok { + hostContextRoots[domain] = map[string]*ContextRoot{} + } + hc, ok := hostContextRoots[domain][contextRootName] + if !ok { + hostContextRoots[domain][contextRootName] = &ContextRoot{ + Name: contextRootName, + Path: contextRoot, + Rewrite: contextRootRewrite, + Upstreams: []string{}, + } + + hc = hostContextRoots[domain][contextRootName] + } + + hc.Upstreams = append(hc.Upstreams, addr) + } + + // "parse" multiple labels for websocket endpoints + websocketEndpoints := utils.WebsocketEndpoints(labels) + + log().Debugf("websocket endpoints: %v", websocketEndpoints) + + // websocket endpoints + for _, ws := range websocketEndpoints { + hostWebsocketEndpoints[domain] = append(hostWebsocketEndpoints[domain], ws) + } + + // "parse" multiple labels for alias domains + aliasDomains := utils.AliasDomains(labels) + + log().Debugf("alias domains: %v", aliasDomains) + + for _, alias := range aliasDomains { + log().Debugf("adding alias %s for %s", alias, srvId) + serverNames[domain] = append(serverNames[domain], alias) + } + + if contextRoot == "" { + log().Debugf("adding upstream %s: upstream=%s", domain, addr) + upstreamServers[domain] = append(upstreamServers[domain], addr) + } + + upstreamHosts[domain] = struct{}{} + log().Infof("%s: upstream=%s", domain, addr) + } + + for k, _ := range upstreamHosts { + log().Debugf("%s contextroots=%+v", k, hostContextRoots[k]) + h := &Host{ + ServerNames: serverNames[k], + Port: p.cfg.Port, + ContextRoots: hostContextRoots[k], + SSLPort: p.cfg.SSLPort, + SSL: hostSSL[k], + SSLCert: hostSSLCert[k], + SSLCertKey: hostSSLCertKey[k], + SSLOnly: hostSSLOnly[k], + SSLBackend: hostSSLBackend[k], + WebsocketEndpoints: hostWebsocketEndpoints[k], + IPHash: hostIPHash[k], + } + + servers := []*Server{} + + for _, s := range upstreamServers[k] { + srv := &Server{ + Addr: s, + } + + servers = append(servers, srv) + } + + up := &Upstream{ + Name: k, + Servers: servers, + } + h.Upstream = up + + hosts = append(hosts, h) + } + + config := &Config{ + Hosts: hosts, + Config: p.cfg, + Networks: networks, + } + + return config, nil +} diff --git a/ext/lb/utils/alias_domains.go b/ext/lb/utils/alias_domains.go index aaf34cc4..4e679bcb 100644 --- a/ext/lb/utils/alias_domains.go +++ b/ext/lb/utils/alias_domains.go @@ -3,14 +3,13 @@ package utils import ( "strings" - "github.com/docker/docker/api/types" "github.com/ehazlett/interlock/ext" ) -func AliasDomains(config types.Container) []string { +func AliasDomains(labels map[string]string) []string { aliasDomains := []string{} - for l, v := range config.Labels { + for l, v := range labels { // this is for labels like interlock.alias_domain.1=foo.local if strings.Index(l, ext.InterlockAliasDomainLabel) > -1 { aliasDomains = append(aliasDomains, v) diff --git a/ext/lb/utils/backend_options.go b/ext/lb/utils/backend_options.go index 3869c95d..5924ed5f 100644 --- a/ext/lb/utils/backend_options.go +++ b/ext/lb/utils/backend_options.go @@ -3,14 +3,13 @@ package utils import ( "strings" - "github.com/docker/docker/api/types" "github.com/ehazlett/interlock/ext" ) -func BackendOptions(config types.Container) []string { +func BackendOptions(labels map[string]string) []string { options := []string{} - for l, v := range config.Labels { + for l, v := range labels { // this is for labels like interlock.backend_option.1=foo if strings.Index(l, ext.InterlockBackendOptionLabel) > -1 { options = append(options, v) diff --git a/ext/lb/utils/balance.go b/ext/lb/utils/balance.go index 35eabd2f..b81d6fa6 100644 --- a/ext/lb/utils/balance.go +++ b/ext/lb/utils/balance.go @@ -1,7 +1,6 @@ package utils import ( - "github.com/docker/docker/api/types" "github.com/ehazlett/interlock/ext" ) @@ -9,10 +8,10 @@ const ( DefaultBalanceAlgorithm = "roundrobin" ) -func BalanceAlgorithm(config types.Container) string { +func BalanceAlgorithm(labels map[string]string) string { algo := DefaultBalanceAlgorithm - if v, ok := config.Labels[ext.InterlockBalanceAlgorithmLabel]; ok { + if v, ok := labels[ext.InterlockBalanceAlgorithmLabel]; ok { algo = v } diff --git a/ext/lb/utils/context_root.go b/ext/lb/utils/context_root.go index a1145828..f0fd58cb 100644 --- a/ext/lb/utils/context_root.go +++ b/ext/lb/utils/context_root.go @@ -1,20 +1,19 @@ package utils import ( - "github.com/docker/docker/api/types" "github.com/ehazlett/interlock/ext" ) -func ContextRoot(config types.Container) string { - if v, ok := config.Labels[ext.InterlockContextRootLabel]; ok { +func ContextRoot(labels map[string]string) string { + if v, ok := labels[ext.InterlockContextRootLabel]; ok { return v } return "" } -func ContextRootRewrite(config types.Container) bool { - if _, ok := config.Labels[ext.InterlockContextRootRewriteLabel]; ok { +func ContextRootRewrite(labels map[string]string) bool { + if _, ok := labels[ext.InterlockContextRootRewriteLabel]; ok { return true } diff --git a/ext/lb/utils/domain.go b/ext/lb/utils/domain.go index fd83fd64..2448e3e8 100644 --- a/ext/lb/utils/domain.go +++ b/ext/lb/utils/domain.go @@ -1,14 +1,13 @@ package utils import ( - "github.com/docker/docker/api/types" "github.com/ehazlett/interlock/ext" ) -func Domain(config types.Container) string { +func Domain(labels map[string]string) string { domain := "local" - if v, ok := config.Labels[ext.InterlockDomainLabel]; ok { + if v, ok := labels[ext.InterlockDomainLabel]; ok { domain = v } diff --git a/ext/lb/utils/health_check.go b/ext/lb/utils/health_check.go index 39ce089d..c3d5239d 100644 --- a/ext/lb/utils/health_check.go +++ b/ext/lb/utils/health_check.go @@ -3,7 +3,6 @@ package utils import ( "strconv" - "github.com/docker/docker/api/types" "github.com/ehazlett/interlock/ext" ) @@ -11,18 +10,18 @@ const ( DefaultHealthCheckInterval = 5000 ) -func HealthCheck(config types.Container) string { - if v, ok := config.Labels[ext.InterlockHealthCheckLabel]; ok { +func HealthCheck(labels map[string]string) string { + if v, ok := labels[ext.InterlockHealthCheckLabel]; ok { return v } return "" } -func HealthCheckInterval(config types.Container) (int, error) { +func HealthCheckInterval(labels map[string]string) (int, error) { checkInterval := DefaultHealthCheckInterval - if v, ok := config.Labels[ext.InterlockHealthCheckIntervalLabel]; ok && v != "" { + if v, ok := labels[ext.InterlockHealthCheckIntervalLabel]; ok && v != "" { i, err := strconv.Atoi(v) if err != nil { return -1, err diff --git a/ext/lb/utils/hostname.go b/ext/lb/utils/hostname.go index e78251fb..8d009c35 100644 --- a/ext/lb/utils/hostname.go +++ b/ext/lb/utils/hostname.go @@ -1,14 +1,13 @@ package utils import ( - "github.com/docker/docker/api/types" "github.com/ehazlett/interlock/ext" ) -func Hostname(config types.Container) string { +func Hostname(labels map[string]string) string { hostname := "unknown" - if v, ok := config.Labels[ext.InterlockHostnameLabel]; ok { + if v, ok := labels[ext.InterlockHostnameLabel]; ok { hostname = v } diff --git a/ext/lb/utils/ip_hash.go b/ext/lb/utils/ip_hash.go index 1c8135cc..c15940b5 100644 --- a/ext/lb/utils/ip_hash.go +++ b/ext/lb/utils/ip_hash.go @@ -1,14 +1,13 @@ package utils import ( - "github.com/docker/docker/api/types" "github.com/ehazlett/interlock/ext" ) -func IPHash(config types.Container) bool { +func IPHash(labels map[string]string) bool { ipHash := false - if _, ok := config.Labels[ext.InterlockIPHashLabel]; ok { + if _, ok := labels[ext.InterlockIPHashLabel]; ok { ipHash = true } diff --git a/ext/lb/utils/network.go b/ext/lb/utils/network.go index ad0aecc4..e65a051c 100644 --- a/ext/lb/utils/network.go +++ b/ext/lb/utils/network.go @@ -6,13 +6,12 @@ import ( "strconv" "github.com/docker/docker/api/types" - ctypes "github.com/docker/docker/api/types" "github.com/docker/go-connections/nat" "github.com/ehazlett/interlock/ext" ) -func OverlayEnabled(config ctypes.Container) (string, bool) { - if v, ok := config.Labels[ext.InterlockNetworkLabel]; ok { +func OverlayEnabled(labels map[string]string) (string, bool) { + if v, ok := labels[ext.InterlockNetworkLabel]; ok { return v, true } @@ -91,3 +90,17 @@ func BackendAddress(cnt types.Container, backendOverrideAddress string) (string, addr = fmt.Sprintf("%s:%s", portDef.HostIP, portDef.HostPort) return addr, nil } + +func CustomPort(labels map[string]string) (int, error) { + // check for custom port + var interlockPort int + if v, ok := labels[ext.InterlockPortLabel]; ok { + var err error + interlockPort, err = strconv.Atoi(v) + if err != nil { + return -1, err + } + } + + return interlockPort, nil +} diff --git a/ext/lb/utils/ssl.go b/ext/lb/utils/ssl.go index 0344dc2f..3b85183a 100644 --- a/ext/lb/utils/ssl.go +++ b/ext/lb/utils/ssl.go @@ -1,7 +1,6 @@ package utils import ( - "github.com/docker/docker/api/types" "github.com/ehazlett/interlock/ext" ) @@ -9,50 +8,50 @@ const ( DefaultSSLBackendTLSVerify = "none" ) -func SSLEnabled(config types.Container) bool { - if _, ok := config.Labels[ext.InterlockSSLLabel]; ok { +func SSLEnabled(labels map[string]string) bool { + if _, ok := labels[ext.InterlockSSLLabel]; ok { return true } return false } -func SSLOnly(config types.Container) bool { - if _, ok := config.Labels[ext.InterlockSSLOnlyLabel]; ok { +func SSLOnly(labels map[string]string) bool { + if _, ok := labels[ext.InterlockSSLOnlyLabel]; ok { return true } return false } -func SSLBackend(config types.Container) bool { - if _, ok := config.Labels[ext.InterlockSSLBackendLabel]; ok { +func SSLBackend(labels map[string]string) bool { + if _, ok := labels[ext.InterlockSSLBackendLabel]; ok { return true } return false } -func SSLCertName(config types.Container) string { - if v, ok := config.Labels[ext.InterlockSSLCertLabel]; ok { +func SSLCertName(labels map[string]string) string { + if v, ok := labels[ext.InterlockSSLCertLabel]; ok { return v } return "" } -func SSLCertKey(config types.Container) string { - if v, ok := config.Labels[ext.InterlockSSLCertKeyLabel]; ok { +func SSLCertKey(labels map[string]string) string { + if v, ok := labels[ext.InterlockSSLCertKeyLabel]; ok { return v } return "" } -func SSLBackendTLSVerify(config types.Container) string { +func SSLBackendTLSVerify(labels map[string]string) string { verify := DefaultSSLBackendTLSVerify - if v, ok := config.Labels[ext.InterlockSSLBackendTLSVerifyLabel]; ok { + if v, ok := labels[ext.InterlockSSLBackendTLSVerifyLabel]; ok { verify = v } diff --git a/ext/lb/utils/websocket.go b/ext/lb/utils/websocket.go index 32357905..9802eae4 100644 --- a/ext/lb/utils/websocket.go +++ b/ext/lb/utils/websocket.go @@ -3,14 +3,13 @@ package utils import ( "strings" - "github.com/docker/docker/api/types" "github.com/ehazlett/interlock/ext" ) -func WebsocketEndpoints(config types.Container) []string { +func WebsocketEndpoints(labels map[string]string) []string { websocketEndpoints := []string{} - for l, v := range config.Labels { + for l, v := range labels { // this is for labels like interlock.websocket_endpoint.1=foo if strings.Index(l, ext.InterlockWebsocketEndpointLabel) > -1 { websocketEndpoints = append(websocketEndpoints, v) From 4208584ec4c0cefe1bf6f046951c3cad13bb2eeb Mon Sep 17 00:00:00 2001 From: jean-claude cote Date: Fri, 8 Sep 2017 10:21:33 -0400 Subject: [PATCH 2/8] integrated ideas from the swarm-services branch --- .../nginx-swarm-stack-machine/README.md | 22 +++- .../docker-compose.yml | 2 +- ext/lb/haproxy/generate.go | 116 ++++++++++++++---- ext/lb/lb.go | 97 +++++++-------- ext/lb/nginx/generate.go | 85 +++++++++++-- 5 files changed, 229 insertions(+), 93 deletions(-) diff --git a/docs/examples/nginx-swarm-stack-machine/README.md b/docs/examples/nginx-swarm-stack-machine/README.md index 57b634ca..76a74014 100644 --- a/docs/examples/nginx-swarm-stack-machine/README.md +++ b/docs/examples/nginx-swarm-stack-machine/README.md @@ -16,12 +16,25 @@ Interlock to your Swarm cluster. Run the following to set it up: `docker-machine env manager` export DOCKER_TLS_VERIFY="1" -export DOCKER_HOST="tcp://192.168.99.102:2376" +export DOCKER_HOST="tcp://192.168.99.100:2376" export DOCKER_CERT_PATH="/Users/jccote/.docker/machine/machines/manager" export DOCKER_MACHINE_NAME="manager" # Run this command to configure your shell: # eval $(docker-machine env manager) +# build an interlock image using a custom tag +# this builds the image into the DOCKER_HOST specified above that is the manager node +`make -e TAG=mytag image` + +# you can verify the image was created in the manager node +`docker image ls` +REPOSITORY TAG IMAGE ID CREATED SIZE +ehazlett/interlock mytag 7b20d1a87b1e 15 seconds ago 23.2MB + 0da48c200507 34 seconds ago 634MB +nginx 0b5dec81616c 44 hours ago 108MB +alpine latest 7328f6f8b418 2 months ago 3.97MB +golang 1.6-alpine 1ea38172de32 8 months ago 283MB + # generate a stack file using docker-compose `docker-compose -f ./docs/examples/nginx-swarm-stack-machine/docker-compose.yml config > stack.yml` @@ -37,11 +50,11 @@ ykdsht0davud mystack_interlock replicated 1/1 Once up you can check the logs to ensure Interlock is detecting: -`docker logs mystack_interlock` +`docker logs mystack_interlock.1.5s9qd89crem17f384o2zt2kv2` You can also verify that the nginx routes are created properly: -`docker exec -it mystack_nginx.1.d2tt5tdwcsz0yq91wjwhympqy /bin/bash -c "cat /etc/nginx/nginx.conf"` +`docker exec -it mystack_nginx.1.5s9qd89crem17f384o2zt2kv2 /bin/bash -c "cat /etc/nginx/nginx.conf"` upstream ctx___web { zone ctx___web_backend 64k; @@ -62,5 +75,4 @@ You can also verify that the nginx routes are created properly: The sample web applications should be available at -http://192.168.99.102/web - +http://192.168.99.100:7070/web diff --git a/docs/examples/nginx-swarm-stack-machine/docker-compose.yml b/docs/examples/nginx-swarm-stack-machine/docker-compose.yml index 2bb37973..3bd47b3e 100644 --- a/docs/examples/nginx-swarm-stack-machine/docker-compose.yml +++ b/docs/examples/nginx-swarm-stack-machine/docker-compose.yml @@ -6,7 +6,7 @@ networks: services: interlock: - image: ehazlett/interlock:jcmcote + image: ehazlett/interlock:mytag command: -D run -c /etc/interlock/config.toml tty: true networks: diff --git a/ext/lb/haproxy/generate.go b/ext/lb/haproxy/generate.go index a52a3c56..5ffd714e 100644 --- a/ext/lb/haproxy/generate.go +++ b/ext/lb/haproxy/generate.go @@ -11,7 +11,7 @@ import ( "net" ) -func (p *HAProxyLoadBalancer) GenerateProxyConfig(containers []types.Container) (interface{}, error) { +func (p *HAProxyLoadBalancer) GenerateProxyConfig(containers []types.Container, tasks []swarm.Task) (interface{}, error) { var hosts []*Host proxyUpstreams := map[string][]*Upstream{} @@ -23,12 +23,35 @@ func (p *HAProxyLoadBalancer) GenerateProxyConfig(containers []types.Container) hostSSLOnly := map[string]bool{} hostSSLBackend := map[string]bool{} hostSSLBackendTLSVerify := map[string]string{} + cntId := "" + labels := map[string]string{} + container_name := "" networks := map[string]string{} - for _, c := range containers { - cntId := c.ID[:12] - labels := c.Labels + var backends []interface{} + + for _, i := range containers { + backends = append(backends, i) + } + + for _, i := range tasks { + backends = append(backends, i) + } + + for _, c := range backends { + switch t := c.(type) { + case types.Container: + cntId = t.ID[:12] + labels = t.Labels + case swarm.Task: + cntId = t.ID[:12] + labels = t.Labels + default: + log().Warnf("unknown type detected: %v", t) + continue + } + // load interlock data hostname := utils.Hostname(labels) domain := utils.Domain(labels) @@ -89,38 +112,81 @@ func (p *HAProxyLoadBalancer) GenerateProxyConfig(containers []types.Container) addr := "" - // check for networking - if n, ok := utils.OverlayEnabled(labels); ok { - log().Debugf("configuring docker network: name=%s", n) + switch t := c.(type) { + case types.Container: - network, err := p.client.NetworkInspect(context.Background(), n, false) - if err != nil { - log().Error(err) - continue - } + // check for networking + if n, ok := utils.OverlayEnabled(labels); ok { + log().Debugf("configuring docker network: name=%s", n) + + network, err := p.client.NetworkInspect(context.Background(), n, false) + if err != nil { + log().Error(err) + continue + } + + addr, err = utils.BackendOverlayAddress(network, t) + if err != nil { + log().Error(err) + continue + } + + networks[n] = "" + } else { + if len(t.Ports) == 0 { + log().Warnf("%s: no ports exposed", cntId) + continue + } - addr, err = utils.BackendOverlayAddress(network, c) + a, err := utils.BackendAddress(t, p.cfg.BackendOverrideAddress) + if err != nil { + log().Error(err) + continue + } + addr = a + } + container_name = t.Names[0][1:] + case swarm.Task: + interlockPort, err := utils.CustomPort(labels) if err != nil { log().Error(err) continue } + log().Debug(interlockPort) - networks[n] = "" - } else { - if len(c.Ports) == 0 { - log().Warnf("%s: no ports exposed", cntId) - continue - } + // check for networking + if overlayNetworkName, ok := utils.OverlayEnabled(labels); ok { + log().Debugf("configuring docker network: name=%s", overlayNetworkName) - a, err := utils.BackendAddress(c, p.cfg.BackendOverrideAddress) - if err != nil { - log().Error(err) - continue + for _, networksAttachment := range t.NetworksAttachments { + + if overlayNetworkName == networksAttachment.Network.Spec.Annotations.Name { + for _, address := range networksAttachment.Addresses { + log().Debug(address) + + ip, _, err := net.ParseCIDR(address) + if err != nil { + log().Error(err) + continue + } + + addr = fmt.Sprintf("%s:%d", ip, interlockPort) + log().Debug(addr) + } + } + } + + networks[overlayNetworkName] = "" + } else { + + //addr = fmt.Sprintf("%s:%d", network, interlockPort) } - addr = a + container_name = ""//t.Names[0][1:] + default: + log().Warnf("unknown type detected: %v", t) + continue } - container_name := c.Names[0][1:] up := &Upstream{ Addr: addr, Container: container_name, diff --git a/ext/lb/lb.go b/ext/lb/lb.go index e4c1ae0d..2e2c0a74 100644 --- a/ext/lb/lb.go +++ b/ext/lb/lb.go @@ -48,8 +48,7 @@ type proxyContainerNetworkConfig struct { type LoadBalancerBackend interface { Name() string ConfigPath() string - GenerateProxyConfig(c []types.Container) (interface{}, error) - GenerateProxyConfigForTasks(s []swarm.Task) (interface{}, error) + GenerateProxyConfig(c []types.Container, s []swarm.Task) (interface{}, error) Template() string Reload(proxyContainers []types.Container) error } @@ -207,64 +206,54 @@ func NewLoadBalancer(c *config.ExtensionConfig, client *client.Client) (*LoadBal var cfg interface{} - if extension.cfg.SwarmTaskMode == true { - optFilters := filters.NewArgs() - // jcc, this filter is not working... - // optFilters.Add("label", "interlock.hostname") - optFilters.Add("desired-state", "running") - opts := types.TaskListOptions{ - Filters: optFilters, - } - log().Debug("getting task list") - tasks, err := client.TaskList(context.Background(), opts) - if err != nil { - errChan <- err - continue - } + optTaskFilters := filters.NewArgs() + // jcc, this filter is not working... + // optFilters.Add("label", "interlock.hostname") + optTaskFilters.Add("desired-state", "running") + optsTask := types.TaskListOptions{ + Filters: optTaskFilters, + } + log().Debug("getting task list") + tasks, err := client.TaskList(context.Background(), optsTask) + if err != nil { + errChan <- err + continue + } - var proxyTasks []swarm.Task - for _, t := range tasks { - labels := t.Spec.ContainerSpec.Labels - hostname := utils.Hostname(labels) - if hostname != "unknown" { - proxyTasks = append(proxyTasks, t) - } - } - // generate proxy config - log().Debug("generating proxy config for tasks") - cfg, err = extension.backend.GenerateProxyConfigForTasks(proxyTasks) - if err != nil { - errChan <- err - continue - } - log().Debugf("service gen conf: %s", cfg) - } else { - optFilters := filters.NewArgs() - optFilters.Add("status", "running") - optFilters.Add("label", "interlock.hostname") - opts := types.ContainerListOptions{ - All: false, - Size: false, - Filters: optFilters, - } - log().Debug("getting container list") - containers, err := client.ContainerList(context.Background(), opts) - if err != nil { - errChan <- err - continue + var proxyTasks []swarm.Task + for _, t := range tasks { + labels := t.Spec.ContainerSpec.Labels + hostname := utils.Hostname(labels) + if hostname != "unknown" { + proxyTasks = append(proxyTasks, t) } + } - // generate proxy config - log().Debug("generating proxy config") - cfg, err = extension.backend.GenerateProxyConfig(containers) - if err != nil { - errChan <- err - continue - } + optFilters := filters.NewArgs() + optFilters.Add("status", "running") + optFilters.Add("label", "interlock.hostname") + opts := types.ContainerListOptions{ + All: false, + Size: false, + Filters: optFilters, + } + log().Debug("getting container list") + containers, err := client.ContainerList(context.Background(), opts) + if err != nil { + errChan <- err + continue + } - log().Debugf("container gen conf: %s", cfg) + // generate proxy config + log().Debug("generating proxy config") + cfg, err = extension.backend.GenerateProxyConfig(containers, tasks) + if err != nil { + errChan <- err + continue } + log().Debugf("gen conf: %s", cfg) + // save proxy config configPath := extension.backend.ConfigPath() log().Debugf("proxy config path: %s", configPath) diff --git a/ext/lb/nginx/generate.go b/ext/lb/nginx/generate.go index 13c2f840..fb02f2ee 100644 --- a/ext/lb/nginx/generate.go +++ b/ext/lb/nginx/generate.go @@ -12,7 +12,7 @@ import ( "net" ) -func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (interface{}, error) { +func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container, tasks [] swarm.Task) (interface{}, error) { var hosts []*Host upstreamHosts := map[string]struct{}{} upstreamServers := map[string][]string{} @@ -26,10 +26,37 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i hostWebsocketEndpoints := map[string][]string{} hostIPHash := map[string]bool{} networks := map[string]string{} + cntId := "" + labels := map[string]string{} + + var backends []interface{} + + for _, i := range containers { + backends = append(backends, i) + } + + for _, i := range tasks { + backends = append(backends, i) + } + + for _, c := range backends { + switch t := c.(type) { + case types.Container: + cntId = t.ID[:12] + labels = t.Labels + case swarm.Task: + cntId = t.ID[:12] + labels = t.Spec.ContainerSpec.Labels + + if t.Status.State != swarm.TaskStateRunning { + continue + } + + default: + log().Warnf("unknown type detected: %v", t) + continue + } - for _, c := range containers { - cntId := c.ID[:12] - labels := c.Labels // load interlock data contextRoot := utils.ContextRoot(labels) @@ -37,7 +64,7 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i domain := utils.Domain(labels) - if domain == "" && contextRoot == "" { + if domain == "local" && contextRoot == "" { continue } @@ -81,6 +108,9 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i addr := "" + switch t := c.(type) { + case types.Container: + // check for networking if n, ok := utils.OverlayEnabled(labels); ok { log().Debugf("configuring docker network: name=%s", n) @@ -91,7 +121,7 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i continue } - addr, err = utils.BackendOverlayAddress(network, c) + addr, err = utils.BackendOverlayAddress(network, t) if err != nil { log().Error(err) continue @@ -99,12 +129,12 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i networks[n] = "" } else { - if len(c.Ports) == 0 { + if len(t.Ports) == 0 { log().Warnf("%s: no ports exposed", cntId) continue } - a, err := utils.BackendAddress(c, p.cfg.BackendOverrideAddress) + a, err := utils.BackendAddress(t, p.cfg.BackendOverrideAddress) if err != nil { log().Error(err) continue @@ -112,6 +142,45 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container) (i addr = a } + case swarm.Task: + interlockPort, err := utils.CustomPort(labels) + if err != nil { + log().Error(err) + continue + } + log().Debug(interlockPort) + + // check for networking + if overlayNetworkName, ok := utils.OverlayEnabled(labels); ok { + log().Debugf("configuring docker network: name=%s", overlayNetworkName) + + for _, networksAttachment := range t.NetworksAttachments { + + if overlayNetworkName == networksAttachment.Network.Spec.Annotations.Name { + for _, address := range networksAttachment.Addresses { + log().Debug(address) + + ip, _, err := net.ParseCIDR(address) + if err != nil { + log().Error(err) + continue + } + + addr = fmt.Sprintf("%s:%d", ip, interlockPort) + log().Debug(addr) + } + } + } + + networks[overlayNetworkName] = "" + } else { + + //addr = fmt.Sprintf("%s:%d", network, interlockPort) + } + default: + log().Warnf("unknown type detected: %v", t) + continue + } if contextRoot != "" { if _, ok := hostContextRoots[domain]; !ok { From edd7983a8843a123321e44019b6cf2525541718f Mon Sep 17 00:00:00 2001 From: jean-claude cote Date: Fri, 8 Sep 2017 19:38:50 -0400 Subject: [PATCH 3/8] Removed SwarmTaskMode flag --- config/config.go | 1 - .../nginx-swarm-stack-machine/config-for-ide.toml | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 docs/examples/nginx-swarm-stack-machine/config-for-ide.toml diff --git a/config/config.go b/config/config.go index 9902a800..4acc35f1 100644 --- a/config/config.go +++ b/config/config.go @@ -22,7 +22,6 @@ type ExtensionConfig struct { MaxConn int // haproxy, nginx Port int // haproxy, nginx SyslogAddr string // haproxy - SwarmTaskMode bool // haproxy, nginx AdminUser string // haproxy AdminPass string // haproxy SSLCertPath string // haproxy, nginx diff --git a/docs/examples/nginx-swarm-stack-machine/config-for-ide.toml b/docs/examples/nginx-swarm-stack-machine/config-for-ide.toml new file mode 100644 index 00000000..6c944a73 --- /dev/null +++ b/docs/examples/nginx-swarm-stack-machine/config-for-ide.toml @@ -0,0 +1,15 @@ +ListenAddr = ":9090" +DockerURL = "tcp://192.168.99.100:2376" +TLSCACert = "/Users/jccote/.docker/machine/machines/manager/ca.pem" +TLSCert = "/Users/jccote/.docker/machine/machines/manager/cert.pem" +TLSKey = "/Users/jccote/.docker/machine/machines/manager/key.pem" + +[[Extensions]] +Name = "nginx" +ConfigPath = "/etc/nginx/nginx.conf" +TemplatePath = "interlock/docs/examples/nginx-swarm-stack-machine/nginx-template.conf" +PidPath = "/var/run/nginx.pid" +BackendOverrideAddress = "172.17.0.1" +MaxConn = 1024 +Port = 7070 +NginxPlusEnabled = false From 42c9e81841ae26c74f13d1a74ce9617bde0306cb Mon Sep 17 00:00:00 2001 From: jean-claude cote Date: Fri, 8 Sep 2017 20:09:09 -0400 Subject: [PATCH 4/8] forgot to un-comment running-inside-docker check --- ext/lb/lb.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/lb/lb.go b/ext/lb/lb.go index 2e2c0a74..e6050e55 100644 --- a/ext/lb/lb.go +++ b/ext/lb/lb.go @@ -111,10 +111,10 @@ func NewLoadBalancer(c *config.ExtensionConfig, client *client.Client) (*LoadBal }) // load containerID for the following nodeID -// containerID, err := utils.GetContainerID() -// if err != nil { -// return nil, err -// } + containerID, err := utils.GetContainerID() + if err != nil { + return nil, err + } containerID := "none" log().Infof("interlock node: container id=%s", containerID) From 8a958ca258854d48b8fc5a8114169c03931f087b Mon Sep 17 00:00:00 2001 From: jean-claude cote Date: Sat, 23 Sep 2017 06:55:19 -0400 Subject: [PATCH 5/8] forgot to remove commented out code --- ext/lb/lb.go | 1 - ext/lb/nginx/generate.go | 203 --------------------------------------- 2 files changed, 204 deletions(-) diff --git a/ext/lb/lb.go b/ext/lb/lb.go index e6050e55..6b06bc3d 100644 --- a/ext/lb/lb.go +++ b/ext/lb/lb.go @@ -116,7 +116,6 @@ func NewLoadBalancer(c *config.ExtensionConfig, client *client.Client) (*LoadBal return nil, err } - containerID := "none" log().Infof("interlock node: container id=%s", containerID) extension := &LoadBalancer{ diff --git a/ext/lb/nginx/generate.go b/ext/lb/nginx/generate.go index fb02f2ee..31db04b6 100644 --- a/ext/lb/nginx/generate.go +++ b/ext/lb/nginx/generate.go @@ -279,206 +279,3 @@ func (p *NginxLoadBalancer) GenerateProxyConfig(containers []types.Container, ta return config, nil } -// JCC -func (p *NginxLoadBalancer) GenerateProxyConfigForTasks(tasks []swarm.Task) (interface{}, error) { - var hosts []*Host - upstreamHosts := map[string]struct{}{} - upstreamServers := map[string][]string{} - serverNames := map[string][]string{} - hostContextRoots := map[string]map[string]*ContextRoot{} - hostSSL := map[string]bool{} - hostSSLCert := map[string]string{} - hostSSLCertKey := map[string]string{} - hostSSLOnly := map[string]bool{} - hostSSLBackend := map[string]bool{} - hostWebsocketEndpoints := map[string][]string{} - hostIPHash := map[string]bool{} - networks := map[string]string{} - - // JCC instead of containers I'll have service specifications here - - for _, t := range tasks { - srvId := t.ID[:12] - labels := t.Spec.ContainerSpec.Labels - // load interlock data - contextRoot := utils.ContextRoot(labels) - - hostname := utils.Hostname(labels) - domain := utils.Domain(labels) - - if t.Status.State != swarm.TaskStateRunning { - continue - } - - if domain == "" && contextRoot == "" { - continue - } - - if hostname != domain && hostname != "" { - domain = fmt.Sprintf("%s.%s", hostname, domain) - } - - // context root - contextRootName := fmt.Sprintf("%s_%s", domain, strings.Replace(contextRoot, "/", "_", -1)) - contextRootRewrite := utils.ContextRootRewrite(labels) - - // check if the first server name is there; if not, add - // this happens if there are multiple backend containers - if _, ok := serverNames[domain]; !ok { - serverNames[domain] = []string{domain} - } - - // JCC many of the checks here are for labels on containers, the labels are on the service definition as well. - - hostSSL[domain] = utils.SSLEnabled(labels) - hostSSLOnly[domain] = utils.SSLOnly(labels) - hostIPHash[domain] = utils.IPHash(labels) - // check ssl backend - hostSSLBackend[domain] = utils.SSLBackend(labels) - - // set cert paths - baseCertPath := p.cfg.SSLCertPath - - certName := utils.SSLCertName(labels) - - if certName != "" { - certPath := filepath.Join(baseCertPath, certName) - log().Infof("ssl cert for %s: %s", domain, certPath) - hostSSLCert[domain] = certPath - } - - certKeyName := utils.SSLCertKey(labels) - if certKeyName != "" { - keyPath := filepath.Join(baseCertPath, certKeyName) - log().Infof("ssl key for %s: %s", domain, keyPath) - hostSSLCertKey[domain] = keyPath - } - - addr := "" - - interlockPort, err := utils.CustomPort(labels) - if err != nil { - log().Error(err) - continue - } - log().Debug(interlockPort) - - // check for networking - if overlayNetworkName, ok := utils.OverlayEnabled(labels); ok { - log().Debugf("configuring docker network: name=%s", overlayNetworkName) - - for _, networksAttachment := range t.NetworksAttachments { - - if overlayNetworkName == networksAttachment.Network.Spec.Annotations.Name { - for _, address := range networksAttachment.Addresses { - log().Debug(address) - - ip, _, err := net.ParseCIDR(address) - if err != nil { - log().Error(err) - continue - } - - addr = fmt.Sprintf("%s:%d", ip, interlockPort) - log().Debug(addr) - } - } - } - - networks[overlayNetworkName] = "" - } else { - - //addr = fmt.Sprintf("%s:%d", network, interlockPort) - } - - if contextRoot != "" { - if _, ok := hostContextRoots[domain]; !ok { - hostContextRoots[domain] = map[string]*ContextRoot{} - } - hc, ok := hostContextRoots[domain][contextRootName] - if !ok { - hostContextRoots[domain][contextRootName] = &ContextRoot{ - Name: contextRootName, - Path: contextRoot, - Rewrite: contextRootRewrite, - Upstreams: []string{}, - } - - hc = hostContextRoots[domain][contextRootName] - } - - hc.Upstreams = append(hc.Upstreams, addr) - } - - // "parse" multiple labels for websocket endpoints - websocketEndpoints := utils.WebsocketEndpoints(labels) - - log().Debugf("websocket endpoints: %v", websocketEndpoints) - - // websocket endpoints - for _, ws := range websocketEndpoints { - hostWebsocketEndpoints[domain] = append(hostWebsocketEndpoints[domain], ws) - } - - // "parse" multiple labels for alias domains - aliasDomains := utils.AliasDomains(labels) - - log().Debugf("alias domains: %v", aliasDomains) - - for _, alias := range aliasDomains { - log().Debugf("adding alias %s for %s", alias, srvId) - serverNames[domain] = append(serverNames[domain], alias) - } - - if contextRoot == "" { - log().Debugf("adding upstream %s: upstream=%s", domain, addr) - upstreamServers[domain] = append(upstreamServers[domain], addr) - } - - upstreamHosts[domain] = struct{}{} - log().Infof("%s: upstream=%s", domain, addr) - } - - for k, _ := range upstreamHosts { - log().Debugf("%s contextroots=%+v", k, hostContextRoots[k]) - h := &Host{ - ServerNames: serverNames[k], - Port: p.cfg.Port, - ContextRoots: hostContextRoots[k], - SSLPort: p.cfg.SSLPort, - SSL: hostSSL[k], - SSLCert: hostSSLCert[k], - SSLCertKey: hostSSLCertKey[k], - SSLOnly: hostSSLOnly[k], - SSLBackend: hostSSLBackend[k], - WebsocketEndpoints: hostWebsocketEndpoints[k], - IPHash: hostIPHash[k], - } - - servers := []*Server{} - - for _, s := range upstreamServers[k] { - srv := &Server{ - Addr: s, - } - - servers = append(servers, srv) - } - - up := &Upstream{ - Name: k, - Servers: servers, - } - h.Upstream = up - - hosts = append(hosts, h) - } - - config := &Config{ - Hosts: hosts, - Config: p.cfg, - Networks: networks, - } - - return config, nil -} From 26430b847f3e24e415e4f95721fb8d46ae9ac998 Mon Sep 17 00:00:00 2001 From: jean-claude cote Date: Sat, 23 Sep 2017 07:08:52 -0400 Subject: [PATCH 6/8] fix import --- ext/lb/lb.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/lb/lb.go b/ext/lb/lb.go index 6b06bc3d..951c7411 100644 --- a/ext/lb/lb.go +++ b/ext/lb/lb.go @@ -23,6 +23,7 @@ import ( "github.com/ehazlett/interlock/ext" "github.com/ehazlett/interlock/ext/lb/nginx" "github.com/ehazlett/interlock/ext/lb/utils" + nutils "github.com/ehazlett/interlock/utils" "github.com/ehazlett/ttlcache" "golang.org/x/net/context" "github.com/ehazlett/interlock/ext/lb/haproxy" @@ -111,7 +112,7 @@ func NewLoadBalancer(c *config.ExtensionConfig, client *client.Client) (*LoadBal }) // load containerID for the following nodeID - containerID, err := utils.GetContainerID() + containerID, err := nutils.GetContainerID() if err != nil { return nil, err } From 9e05ad4225fc58d5c044d7d535cd810a1d0a4b25 Mon Sep 17 00:00:00 2001 From: jean-claude cote Date: Sat, 23 Sep 2017 14:33:05 -0400 Subject: [PATCH 7/8] fix test cases --- ext/lb/utils/alias_domains_test.go | 4 ++-- ext/lb/utils/backend_options_test.go | 4 ++-- ext/lb/utils/balance_test.go | 4 ++-- ext/lb/utils/context_root_test.go | 4 ++-- ext/lb/utils/domain_test.go | 2 +- ext/lb/utils/health_check_test.go | 8 ++++---- ext/lb/utils/hostname_test.go | 2 +- ext/lb/utils/network_test.go | 4 ++-- ext/lb/utils/ssl_test.go | 20 ++++++++++---------- ext/lb/utils/websocket_test.go | 4 ++-- 10 files changed, 28 insertions(+), 28 deletions(-) diff --git a/ext/lb/utils/alias_domains_test.go b/ext/lb/utils/alias_domains_test.go index a1ccce18..70a1f7ac 100644 --- a/ext/lb/utils/alias_domains_test.go +++ b/ext/lb/utils/alias_domains_test.go @@ -15,7 +15,7 @@ func TestAliasDomains(t *testing.T) { }, } - ep := AliasDomains(cfg) + ep := AliasDomains(cfg.Labels) if len(ep) != 2 { t.Fatalf("expected %d alias domains; received %d", len(cfg.Labels), len(ep)) @@ -27,7 +27,7 @@ func TestAliasDomainsNoLabels(t *testing.T) { Labels: map[string]string{}, } - ep := AliasDomains(cfg) + ep := AliasDomains(cfg.Labels) if len(ep) != 0 { t.Fatalf("expected no alias domains; received %s", ep) diff --git a/ext/lb/utils/backend_options_test.go b/ext/lb/utils/backend_options_test.go index 3a3b7e4f..300f3280 100644 --- a/ext/lb/utils/backend_options_test.go +++ b/ext/lb/utils/backend_options_test.go @@ -15,7 +15,7 @@ func TestBackendOptions(t *testing.T) { }, } - opts := BackendOptions(cfg) + opts := BackendOptions(cfg.Labels) if len(opts) != 2 { t.Fatalf("expected %d options; received %d", len(cfg.Labels), len(opts)) @@ -27,7 +27,7 @@ func TestBackendOptionsNoLabels(t *testing.T) { Labels: map[string]string{}, } - opts := BackendOptions(cfg) + opts := BackendOptions(cfg.Labels) if len(opts) != 0 { t.Fatalf("expected no options; received %s", opts) diff --git a/ext/lb/utils/balance_test.go b/ext/lb/utils/balance_test.go index 942f2043..3943d98e 100644 --- a/ext/lb/utils/balance_test.go +++ b/ext/lb/utils/balance_test.go @@ -16,7 +16,7 @@ func TestBalanceAlgorithm(t *testing.T) { }, } - algo := BalanceAlgorithm(cfg) + algo := BalanceAlgorithm(cfg.Labels) if algo != testAlgo { t.Fatalf("expected %s; received %s", testAlgo, algo) @@ -28,7 +28,7 @@ func TestBalanceAlgorithmDefault(t *testing.T) { Labels: map[string]string{}, } - algo := BalanceAlgorithm(cfg) + algo := BalanceAlgorithm(cfg.Labels) if algo != DefaultBalanceAlgorithm { t.Fatalf("expected %s; received %s", DefaultBalanceAlgorithm, algo) diff --git a/ext/lb/utils/context_root_test.go b/ext/lb/utils/context_root_test.go index 02eaae59..056a87fb 100644 --- a/ext/lb/utils/context_root_test.go +++ b/ext/lb/utils/context_root_test.go @@ -16,7 +16,7 @@ func TestContextRoot(t *testing.T) { }, } - context := ContextRoot(cfg) + context := ContextRoot(cfg.Labels) if context != testContext { t.Fatalf("expected %s; received %s", testContext, context) @@ -30,7 +30,7 @@ func TestContextRootRewrite(t *testing.T) { }, } - rewrite := ContextRootRewrite(cfg) + rewrite := ContextRootRewrite(cfg.Labels) if !rewrite { t.Fatal("expected context root rewrite") diff --git a/ext/lb/utils/domain_test.go b/ext/lb/utils/domain_test.go index af65d2ae..0d350632 100644 --- a/ext/lb/utils/domain_test.go +++ b/ext/lb/utils/domain_test.go @@ -15,7 +15,7 @@ func TestDomain(t *testing.T) { }, } - domain := Domain(cfg) + domain := Domain(cfg.Labels) if domain != testDomain { t.Fatalf("expected %s; received %s", testDomain, domain) diff --git a/ext/lb/utils/health_check_test.go b/ext/lb/utils/health_check_test.go index 08d3bdaf..2baa6817 100644 --- a/ext/lb/utils/health_check_test.go +++ b/ext/lb/utils/health_check_test.go @@ -17,7 +17,7 @@ func TestHealthCheck(t *testing.T) { }, } - chk := HealthCheck(cfg) + chk := HealthCheck(cfg.Labels) if chk != testCheck { t.Fatalf("expected %s; received %s", testCheck, chk) @@ -29,7 +29,7 @@ func TestHealthCheckNoLabel(t *testing.T) { Labels: map[string]string{}, } - chk := HealthCheck(cfg) + chk := HealthCheck(cfg.Labels) if chk != "" { t.Fatalf("expected no health check; received %s", chk) @@ -45,7 +45,7 @@ func TestHealthCheckInterval(t *testing.T) { }, } - i, err := HealthCheckInterval(cfg) + i, err := HealthCheckInterval(cfg.Labels) if err != nil { t.Fatal(err) } @@ -60,7 +60,7 @@ func TestHealthCheckIntervalNoLabel(t *testing.T) { Labels: map[string]string{}, } - i, err := HealthCheckInterval(cfg) + i, err := HealthCheckInterval(cfg.Labels) if err != nil { t.Fatal(err) } diff --git a/ext/lb/utils/hostname_test.go b/ext/lb/utils/hostname_test.go index 92665a6c..2c11e3cd 100644 --- a/ext/lb/utils/hostname_test.go +++ b/ext/lb/utils/hostname_test.go @@ -16,7 +16,7 @@ func TestHostnameLabel(t *testing.T) { }, } - hostname := Hostname(cfg) + hostname := Hostname(cfg.Labels) if hostname != testLabelHostname { t.Fatalf("expected %s; received %s", testLabelHostname, hostname) diff --git a/ext/lb/utils/network_test.go b/ext/lb/utils/network_test.go index aef6f077..3a95f63e 100644 --- a/ext/lb/utils/network_test.go +++ b/ext/lb/utils/network_test.go @@ -14,7 +14,7 @@ func TestUseOverlay(t *testing.T) { }, } - if _, ok := OverlayEnabled(cfg); !ok { + if _, ok := OverlayEnabled(cfg.Labels); !ok { t.Fatal("expected to use overlay networking") } } @@ -24,7 +24,7 @@ func TestUseOverlayNoLabel(t *testing.T) { Labels: map[string]string{}, } - if _, ok := OverlayEnabled(cfg); ok { + if _, ok := OverlayEnabled(cfg.Labels); ok { t.Fatal("expected to use bridge networking") } } diff --git a/ext/lb/utils/ssl_test.go b/ext/lb/utils/ssl_test.go index 64f2976f..cf817757 100644 --- a/ext/lb/utils/ssl_test.go +++ b/ext/lb/utils/ssl_test.go @@ -14,7 +14,7 @@ func TestSSLEnabled(t *testing.T) { }, } - if !SSLEnabled(cfg) { + if !SSLEnabled(cfg.Labels) { t.Fatal("expected ssl enabled") } } @@ -24,7 +24,7 @@ func TestSSLEnabledNoLabel(t *testing.T) { Labels: map[string]string{}, } - if SSLEnabled(cfg) { + if SSLEnabled(cfg.Labels) { t.Fatal("expected ssl disabled") } } @@ -36,7 +36,7 @@ func TestSSLOnly(t *testing.T) { }, } - if !SSLOnly(cfg) { + if !SSLOnly(cfg.Labels) { t.Fatal("expected ssl only") } } @@ -46,7 +46,7 @@ func TestSSLOnlyNoLabel(t *testing.T) { Labels: map[string]string{}, } - if SSLOnly(cfg) { + if SSLOnly(cfg.Labels) { t.Fatal("expected not ssl only") } } @@ -58,7 +58,7 @@ func TestSSLBackend(t *testing.T) { }, } - if !SSLBackend(cfg) { + if !SSLBackend(cfg.Labels) { t.Fatal("expected ssl backend") } } @@ -68,7 +68,7 @@ func TestSSLBackendNoLabel(t *testing.T) { Labels: map[string]string{}, } - if SSLBackend(cfg) { + if SSLBackend(cfg.Labels) { t.Fatal("expected no ssl backend") } } @@ -82,7 +82,7 @@ func TestSSLCertName(t *testing.T) { }, } - if SSLCertName(cfg) != testCert { + if SSLCertName(cfg.Labels) != testCert { t.Fatalf("expected ssl cert %s", testCert) } } @@ -92,7 +92,7 @@ func TestSSLCertNameNoLabel(t *testing.T) { Labels: map[string]string{}, } - if SSLCertName(cfg) != "" { + if SSLCertName(cfg.Labels) != "" { t.Fatal("expected no ssl cert") } } @@ -106,7 +106,7 @@ func TestSSLCertKey(t *testing.T) { }, } - if SSLCertKey(cfg) != testKey { + if SSLCertKey(cfg.Labels) != testKey { t.Fatalf("expected ssl key %s", testKey) } } @@ -116,7 +116,7 @@ func TestSSLCertKeyNoLabel(t *testing.T) { Labels: map[string]string{}, } - if SSLCertKey(cfg) != "" { + if SSLCertKey(cfg.Labels) != "" { t.Fatal("expected no ssl key") } } diff --git a/ext/lb/utils/websocket_test.go b/ext/lb/utils/websocket_test.go index 0c367d12..faa395d3 100644 --- a/ext/lb/utils/websocket_test.go +++ b/ext/lb/utils/websocket_test.go @@ -15,7 +15,7 @@ func TestWebsocketEndpoints(t *testing.T) { }, } - ep := WebsocketEndpoints(cfg) + ep := WebsocketEndpoints(cfg.Labels) if len(ep) != 2 { t.Fatalf("expected %d endpoints; received %d", len(cfg.Labels), len(ep)) @@ -27,7 +27,7 @@ func TestWebsocketEndpointsNoLabels(t *testing.T) { Labels: map[string]string{}, } - ep := WebsocketEndpoints(cfg) + ep := WebsocketEndpoints(cfg.Labels) if len(ep) != 0 { t.Fatalf("expected no endpoints; received %s", ep) From c5cb67208a62e7dc9674f90cd9c4472221ec318b Mon Sep 17 00:00:00 2001 From: jean-claude cote Date: Mon, 23 Oct 2017 21:42:30 -0400 Subject: [PATCH 8/8] poller checks when tasks state change and triggers and update --- server/server.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/server/server.go b/server/server.go index 3f8f990c..4c53a961 100644 --- a/server/server.go +++ b/server/server.go @@ -13,6 +13,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" etypes "github.com/docker/docker/api/types/events" + "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" "github.com/ehazlett/interlock/config" @@ -22,6 +23,7 @@ import ( "github.com/ehazlett/interlock/ext/lb" "github.com/prometheus/client_golang/prometheus" "golang.org/x/net/context" + "github.com/ehazlett/interlock/ext/lb/utils" ) const ( @@ -257,6 +259,33 @@ func (s *Server) runPoller(d time.Duration) { go func() { for range t.C { log.Debug("poller tick") + + optTaskFilters := filters.NewArgs() + // jcc, this filter is not working... + // optFilters.Add("label", "interlock.hostname") + optTaskFilters.Add("desired-state", "running") + optsTask := types.TaskListOptions{ + Filters: optTaskFilters, + } + log.Debug("getting task list") + tasks, err := s.client.TaskList(context.Background(), optsTask) + if err != nil { + log.Warnf("poller: unable to get tasks: %s", err) + continue + } + + var proxyTasks []swarm.Task + for _, t := range tasks { + labels := t.Spec.ContainerSpec.Labels + hostname := utils.Hostname(labels) + if t.Status.State == swarm.TaskStateRunning { + if hostname != "unknown" { + proxyTasks = append(proxyTasks, t) + } + } + } + + optFilters := filters.NewArgs() optFilters.Add("status", "running") opts := types.ContainerListOptions{ @@ -271,8 +300,13 @@ func (s *Server) runPoller(d time.Duration) { } containerIDs := []string{} + taskIDs := []string{} ports := []int{} + for _, t := range proxyTasks { + taskIDs = append(taskIDs, t.ID) + } + for _, c := range containers { containerIDs = append(containerIDs, c.ID) for _, p := range c.Ports { @@ -280,9 +314,16 @@ func (s *Server) runPoller(d time.Duration) { } } + sort.Strings(taskIDs) sort.Strings(containerIDs) sort.Ints(ports) + tData, err := json.Marshal(taskIDs) + if err != nil { + log.Errorf("unable to marshal containers: %s", err) + continue + } + cData, err := json.Marshal(containerIDs) if err != nil { log.Errorf("unable to marshal containers: %s", err) @@ -296,6 +337,7 @@ func (s *Server) runPoller(d time.Duration) { } h := sha256.New() + h.Write(tData) h.Write(cData) h.Write(pData) sum := hex.EncodeToString(h.Sum(nil))