kubernetes container manager

README.md

@@ -29,3 +29,8 @@ To do all the make steps:
 
     make
 
+
+
+If both the BOSH release and the k8s release are going to be hooked up
+to the same service-broker platform, `services/eth.json` needs to be
+configured so that the class and plan identities do not collide.

cmd/broker/main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"strconv"
 
 	"code.cloudfoundry.org/lager"
 	"github.com/docker/docker/client"
@@ -55,7 +56,7 @@ func main() {
 		if err != nil {
 			logger.Fatal("could not set up a kubernetes-client", err)
 		}
-		manager = kcm.NewKubernetesContainerManager(logger, clientset)
+		manager = kcm.NewKubernetesContainerManager(logger, clientset, state.Config.ExternalAddress)
 		logger.Debug("using kubernetes containermanager")
 	default:
 		logger.Fatal("no container manager in config", fmt.Errorf("no container manager specified in config %q", configFilepath))
@@ -79,6 +80,6 @@ func main() {
 	}
 
 	http.Handle("/", log(brokerAPI))
-	logger.Debug("listening on" + string(state.Config.Port))
+	logger.Debug("listening on port:" + strconv.Itoa(int(state.Config.Port)))
 	logger.Fatal("http-listen", http.ListenAndServe(fmt.Sprintf(":%d", state.Config.Port), nil))
 }

images/Dockerfile.broker

@@ -1,7 +1,7 @@
 from golang:1.11-alpine3.8 as builder
 workdir /go/src/github.com/cloudfoundry-incubator/blockhead
 copy . .
-run go build -v -o /broker ./cmd/broker
+run CGO_ENABLED=0 go build -ldflags '-s -w -d' -v -o /broker ./cmd/broker
 
 
 from node:10

k8s/README.md

@@ -0,0 +1,159 @@
+This directory has the resources to install the broker in k8s. Only need to be
+used once on startup.
+
+`rbac.yaml` sets up the default namespace so that the broker has the access
+needed to create k8s objects.
+
+`deploy.yaml` installs the broker and creates a service to access it in-cluster.
+
+`ingress.yaml` sets up the basic ingress for accessing the broker from outside
+the cluster by a resolving dns name. It needs to be edited to have the right
+host as the ingress domain.
+
+`n.yaml` has test resources to troubleshoot ingress or other networking with a
+very basic nginx setup.
+
+* How to K8s
+
+Prereqs:
+ - get a kube cluster
+ - know an address to access it (it's load-balancer or worker-node)
+
+RBAC setup. We only require access to create and delete services and pods. The
+provided rbac has much more access because I am lazy.
+
+create it with :
+
+    kubectl create -f rbac.yaml
+
+Edit the `deploy.yaml` to fixup the `external_address` field so that it points
+at either the cluster loadbalancer or a publicly accessible node address (IP or DNS).
+Now create the Broker, it's Service, and the config data necessary to start it with:
+
+    kubectl create -f deploy.yaml
+
+Wait a bit and everything should be deployed.
+
+    kubectl get all -l app=blockhead-broker
+
+In that should be the service and what nodeport it is bound to.
+
+We can get the catalog with curl.
+
+```
+curl -sSL  a:b@peanuts.sng01.containers.appdomain.cloud:32089/v2/catalog  -H
+"X-Broker-API-Version: 2.14"
+
+{"services":[{"id":"0d25f970-899a-4aa4-b753-640e33b66389","name":"eth","description":"Ethereum
+Geth
+Node","bindable":true,"tags":["eth","geth","dev"],"plan_updateable":false,"plans":[{"id":"0a20f765-9e22-4ac2-b9f7-2ac8f706747c","name":"free","description":"Free
+Trial","free":true}],"metadata":{"displayName":"Geth 1.8"}}]}
+```
+
+Do a service provision:
+
+```console
+$ curl -sSL  a:b@peanuts.sng01.containers.appdomain.cloud:30000/v2/service_instances/test-broker -d "@sp.json" -XPUT -H "X-Broker-API-Version: 2.14"   -H "Content-Type: application/json"
+```
+where `sp.json` contains the service and plan to provision.
+
+
+Do a bind:
+
+```console
+$ curl http://a:b@peanuts.sng01.containers.appdomain.cloud:30000/v2/service_instances/test-broker/service_bindings/newbind -d "@sp2.json" -XPUT -H "X-Broker-API-Version: 2.14"   -H "Content-Type: application/json"
+{"credentials":{"ContainerInfo":{"ExternalAddress":"peanuts.sng01.containers.appdomain.cloud","InternalAddress":"test-broker","Bindings":{"8545":[{"Port":"30971"}]}},"NodeInfo":{"address":"0x6361eFC13f0b5f0072e3774254a1e9a876db6BD7","abi":"[{\"constant\":false,\"inputs\":[],\"name\":\"voteA\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"proposalA\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[],\"name\":\"voteB\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"proposalB\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"name\":\"_\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"}]","contract_address":"0xA5948eDC459d0D3e83D6f9b9Cf089182EBE2b056","gas_price":"1","transaction_hash":"0xfa6d497809439d305cd64073de44c8e4b51ebc9d9a24fc1a651bd73a6f0b85c7"}}}
+```
+Where `sp2.json` contains the service, plan, and parameters.
+
+If attaching remix from the webconsole, make sure that it's loaded from the insecure http backend.
+
+* cleanup tips
+
+Delete all the deployed geth nodes in one shot
+
+    kubectl delete all -l provisionedBy=blockhead-broker
+
+Delete all the broker stuff in one shot
+
+    kubectl delete all -l app=blockhead-broker
+
+* optional stuff
+
+using ingress is nice. Easiest to be used for the broker. The host role in `ingress.yaml` has to be adjusted.
+
+    kubectl create -f ingress.yaml
+
+Can of course deploy the app in kubernetes.
+
+
+* set up an IKS cluster
+
+On IKS, your loadbalancer DNS entry is found in the output of:
+
+```console
+$ bx cs cluster-get  mhb-blockhead-broker
+Retrieving cluster mhb-blockhead-broker...
+OK
+
+
+Name:                   mhb-blockhead-broker   
+ID:                     26760c5d3aca48619e4b79587feb1ce2   
+State:                  normal   
+Created:                2018-09-29T01:47:10+0000   
+Location:               sao01   
+Master URL:             https://169.57.151.10:31423   
+Master Location:        sao01   
+Master Status:          Ready (2 days ago)   
+Ingress Subdomain:      mhb-blockhead-broker.sao01.containers.appdomain.cloud   
+Ingress Secret:         mhb-blockhead-broker   
+Workers:                1   
+Worker Zones:           sao01   
+Version:                1.11.3_1524   
+Owner:                  mbauer@us.ibm.com   
+Monitoring Dashboard:   -   
+```
+
+Set a region if necessary:
+
+     bx cs region-set ap-north
+
+Look for *Ingress Subdomain*, ours is `mhb-blockhead-broker.sao01.containers.appdomain.cloud`
+
+```console
+$ bx cs machine-types sng01
+OK
+Name        Cores   Memory   Network Speed   OS             Server Type   Storage   Secondary Storage   Trustable   
+u2c.2x4     2       4GB      1000Mbps        UBUNTU_16_64   virtual       25GB      100GB               false   
+c2c.16x16   16      16GB     1000Mbps        UBUNTU_16_64   virtual       25GB      100GB               false   
+c2c.16x32   16      32GB     1000Mbps        UBUNTU_16_64   virtual       25GB      100GB               false   
+b2c.4x16    4       16GB     1000Mbps        UBUNTU_16_64   virtual       25GB      100GB               false   
+b2c.8x32    8       32GB     1000Mbps        UBUNTU_16_64   virtual       25GB      100GB               false   
+b2c.16x64   16      64GB     1000Mbps        UBUNTU_16_64   virtual       25GB      100GB               false   
+```
+
+
+look up the vlans to add
+
+```
+$ bx cs vlans --zone sng01
+OK
+ID        Name   Number   Type      Router         Supports Virtual Workers   
+2457919          1458     private   bcr01a.sng01   true   
+2457917          1406     public    fcr01a.sng01   true   
+```
+
+```
+$ bx cs cluster-create --name peanuts --kube-version 1.11.3 --location sng01  --machine-type u2c.2x4 --workers 3  --public-vlan 2457917 --private-vlan 2457919
+Creating cluster...
+OK
+```
+
+Eventually...
+
+```
+$ bx cs clusters
+OK
+Name      ID                                 State       Created          Workers   Location   Version   
+peanuts   ed40c720877447e192fb51e09f7e4474   requested   30 seconds ago   1         sng01      1.11.3_1524   
+```

k8s/deploy.yaml

@@ -0,0 +1,62 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: blockhead-broker
+  namespace: default
+data:
+  config.json: |+
+    {
+        "username":"a",
+        "password":"b",
+        "port": 3333,
+        "deployer_path":"/pusher.js",
+        "container_manager": "kubernetes",
+        "external_address": "peanuts.sng01.containers.appdomain.cloud"
+    }
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: blockhead-broker
+  labels:
+    app: blockhead-broker
+spec:
+  selector:
+    matchLabels:
+      app: blockhead-broker
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: blockhead-broker
+    spec:
+      containers:
+      - name: broker
+        image: mhbauer/blockhead-broker:eu2018
+        command: ["/broker","/config/config.json","/services"]
+        ports:
+        - containerPort: 3333
+        imagePullPolicy: Always
+        volumeMounts:
+          - name: config
+            mountPath: "/config"
+      volumes:
+        - name: config
+          configMap:
+            name: blockhead-broker
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: blockhead-broker
+  name: blockhead-broker
+  namespace: default
+spec:
+  ports:
+  - port: 3333
+    nodePort: 30000
+  selector:
+    app: blockhead-broker
+  type: NodePort

k8s/ingress.yaml

@@ -0,0 +1,16 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: blockhead-broker
+  namespace: default
+  annotations:
+    ingress.bluemix.net/rewrite-path: "serviceName=blockhead-broker rewrite=/"
+spec:
+  rules:
+  - host: mhb-blockhead-broker.sao01.containers.appdomain.cloud
+    http:
+      paths:
+      - path: "/broker/"
+        backend:
+          serviceName: "blockhead-broker"
+          servicePort: 3333

k8s/n.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    run: "n"
+  name: "n"
+spec:
+  containers:
+  - image: nginx
+    name: "n"
+    ports:
+    - containerPort: 80
+  restartPolicy: Never
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    run: "n"
+  name: "n"
+spec:
+  ports:
+  - port: 8080
+    targetPort: 80
+  selector:
+    run: "n"
+  type: NodePort
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: "n"
+  namespace: default
+  annotations:
+    ingress.bluemix.net/rewrite-path: "serviceName=n rewrite=/"
+spec:
+  rules:
+  - host: peanuts.sng01.containers.appdomain.cloud
+    http:
+      paths:
+      - path: "/nginx/"
+        backend:
+          serviceName: "n"
+          servicePort: 8080

k8s/rbac.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: blockhead
+rules:
+  - apiGroups: [""]
+    resources: ["pods","services"]
+    verbs: ["get","create","delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: default-blockhead
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: blockhead
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: default
+

k8s/service-cm.yaml

@@ -0,0 +1,26 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: blockhead-broker-services
+  namespace: default
+data:
+  eth.json: |+
+    {
+        "name": "keth",
+        "description": "Ethereum Geth Node",
+        "tags": [
+            "eth",
+            "geth",
+            "dev"
+        ],
+        "display_name": "Geth 1.8",
+        "plans": [
+            {
+                "name": "free",
+                "free": true,
+                "image": "mhbauer/blockhead-geth:eu2018",
+                "description": "Free Trial",
+                "ports": ["8545"]
+            }
+        ]
+    }

pkg/containermanager/container_manager.go

@@ -19,10 +19,13 @@ type Binding struct {
 	Port string
 }
 
+// internal port to ip:externalport
 type ContainerInfo struct {
 	ExternalAddress string
+	// used for internal communication of broker to container
 	InternalAddress string
-	Bindings        map[string][]Binding
+	// Bindings is a map of container ports to exposed host & port
+	Bindings map[string][]Binding
 }
 
 //go:generate counterfeiter -o ../fakes/fake_container_manager.go . ContainerManager