From 54dded104643fac29c19700c5df7bf8d318632bb Mon Sep 17 00:00:00 2001 From: Morgan Bauer Date: Fri, 28 Sep 2018 18:57:23 -0400 Subject: [PATCH] a kubernetes based container manager comments on how to ingress nodeport service config bind remove ingress static build so I don't have to worry about running things also it's smaller might as well be a nodeport comments on what things are use of config configmap for config info for k8s backend configmap can't be mounted at root, as it will overwrite whole FS mount under subdir and adjust command to load config from that directory pusher config and correct slice temporarily change return values maybe mount service definitions as well fixup rebase changes start of instructions more instructions more instructions and try and make it work again internal binding for internal address missing test file specific rbac prepend an arbitrary character before service use of instance id B, for Blockhead! --- README.md | 5 + cmd/broker/main.go | 5 +- images/Dockerfile.broker | 2 +- k8s/README.md | 159 ++++++++++++++++++ k8s/deploy.yaml | 62 +++++++ k8s/ingress.yaml | 16 ++ k8s/n.yaml | 45 +++++ k8s/rbac.yaml | 23 +++ k8s/service-cm.yaml | 26 +++ pkg/containermanager/container_manager.go | 5 +- .../kubernetes/kubernetes_manager.go | 123 +++++++++++++- pkg/deployer/deployer.go | 4 +- vendor/k8s.io/client-go/README.md | 2 +- 13 files changed, 466 insertions(+), 11 deletions(-) create mode 100644 k8s/README.md create mode 100644 k8s/deploy.yaml create mode 100644 k8s/ingress.yaml create mode 100644 k8s/n.yaml create mode 100644 k8s/rbac.yaml create mode 100644 k8s/service-cm.yaml diff --git a/README.md b/README.md index a73ab06e..d2dbe495 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,8 @@ To do all the make steps: make + + +If both the BOSH release and the k8s release are going to be hooked up +to the same service-broker platform, `services/eth.json` needs to be +configured so that the class and plan identities do not collide. diff --git a/cmd/broker/main.go b/cmd/broker/main.go index 850afa14..6c4fe23b 100644 --- a/cmd/broker/main.go +++ b/cmd/broker/main.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "os" + "strconv" "code.cloudfoundry.org/lager" "github.com/docker/docker/client" @@ -55,7 +56,7 @@ func main() { if err != nil { logger.Fatal("could not set up a kubernetes-client", err) } - manager = kcm.NewKubernetesContainerManager(logger, clientset) + manager = kcm.NewKubernetesContainerManager(logger, clientset, state.Config.ExternalAddress) logger.Debug("using kubernetes containermanager") default: logger.Fatal("no container manager in config", fmt.Errorf("no container manager specified in config %q", configFilepath)) @@ -79,6 +80,6 @@ func main() { } http.Handle("/", log(brokerAPI)) - logger.Debug("listening on" + string(state.Config.Port)) + logger.Debug("listening on port:" + strconv.Itoa(int(state.Config.Port))) logger.Fatal("http-listen", http.ListenAndServe(fmt.Sprintf(":%d", state.Config.Port), nil)) } diff --git a/images/Dockerfile.broker b/images/Dockerfile.broker index f3258753..684e9258 100644 --- a/images/Dockerfile.broker +++ b/images/Dockerfile.broker @@ -1,7 +1,7 @@ from golang:1.11-alpine3.8 as builder workdir /go/src/github.com/cloudfoundry-incubator/blockhead copy . . -run go build -v -o /broker ./cmd/broker +run CGO_ENABLED=0 go build -ldflags '-s -w -d' -v -o /broker ./cmd/broker from node:10 diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 00000000..a0cd6119 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,159 @@ +This directory has the resources to install the broker in k8s. Only need to be +used once on startup. + +`rbac.yaml` sets up the default namespace so that the broker has the access +needed to create k8s objects. + +`deploy.yaml` installs the broker and creates a service to access it in-cluster. + +`ingress.yaml` sets up the basic ingress for accessing the broker from outside +the cluster by a resolving dns name. It needs to be edited to have the right +host as the ingress domain. + +`n.yaml` has test resources to troubleshoot ingress or other networking with a +very basic nginx setup. + +* How to K8s + +Prereqs: + - get a kube cluster + - know an address to access it (it's load-balancer or worker-node) + +RBAC setup. We only require access to create and delete services and pods. The +provided rbac has much more access because I am lazy. + +create it with : + + kubectl create -f rbac.yaml + +Edit the `deploy.yaml` to fixup the `external_address` field so that it points +at either the cluster loadbalancer or a publicly accessible node address (IP or DNS). +Now create the Broker, it's Service, and the config data necessary to start it with: + + kubectl create -f deploy.yaml + +Wait a bit and everything should be deployed. + + kubectl get all -l app=blockhead-broker + +In that should be the service and what nodeport it is bound to. + +We can get the catalog with curl. + +``` +curl -sSL a:b@peanuts.sng01.containers.appdomain.cloud:32089/v2/catalog -H +"X-Broker-API-Version: 2.14" + +{"services":[{"id":"0d25f970-899a-4aa4-b753-640e33b66389","name":"eth","description":"Ethereum +Geth +Node","bindable":true,"tags":["eth","geth","dev"],"plan_updateable":false,"plans":[{"id":"0a20f765-9e22-4ac2-b9f7-2ac8f706747c","name":"free","description":"Free +Trial","free":true}],"metadata":{"displayName":"Geth 1.8"}}]} +``` + +Do a service provision: + +```console +$ curl -sSL a:b@peanuts.sng01.containers.appdomain.cloud:30000/v2/service_instances/test-broker -d "@sp.json" -XPUT -H "X-Broker-API-Version: 2.14" -H "Content-Type: application/json" +``` +where `sp.json` contains the service and plan to provision. + + +Do a bind: + +```console +$ curl http://a:b@peanuts.sng01.containers.appdomain.cloud:30000/v2/service_instances/test-broker/service_bindings/newbind -d "@sp2.json" -XPUT -H "X-Broker-API-Version: 2.14" -H "Content-Type: application/json" +{"credentials":{"ContainerInfo":{"ExternalAddress":"peanuts.sng01.containers.appdomain.cloud","InternalAddress":"test-broker","Bindings":{"8545":[{"Port":"30971"}]}},"NodeInfo":{"address":"0x6361eFC13f0b5f0072e3774254a1e9a876db6BD7","abi":"[{\"constant\":false,\"inputs\":[],\"name\":\"voteA\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"proposalA\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[],\"name\":\"voteB\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"proposalB\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"name\":\"_\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"}]","contract_address":"0xA5948eDC459d0D3e83D6f9b9Cf089182EBE2b056","gas_price":"1","transaction_hash":"0xfa6d497809439d305cd64073de44c8e4b51ebc9d9a24fc1a651bd73a6f0b85c7"}}} +``` +Where `sp2.json` contains the service, plan, and parameters. + +If attaching remix from the webconsole, make sure that it's loaded from the insecure http backend. + +* cleanup tips + +Delete all the deployed geth nodes in one shot + + kubectl delete all -l provisionedBy=blockhead-broker + +Delete all the broker stuff in one shot + + kubectl delete all -l app=blockhead-broker + +* optional stuff + +using ingress is nice. Easiest to be used for the broker. The host role in `ingress.yaml` has to be adjusted. + + kubectl create -f ingress.yaml + +Can of course deploy the app in kubernetes. + + +* set up an IKS cluster + +On IKS, your loadbalancer DNS entry is found in the output of: + +```console +$ bx cs cluster-get mhb-blockhead-broker +Retrieving cluster mhb-blockhead-broker... +OK + + +Name: mhb-blockhead-broker +ID: 26760c5d3aca48619e4b79587feb1ce2 +State: normal +Created: 2018-09-29T01:47:10+0000 +Location: sao01 +Master URL: https://169.57.151.10:31423 +Master Location: sao01 +Master Status: Ready (2 days ago) +Ingress Subdomain: mhb-blockhead-broker.sao01.containers.appdomain.cloud +Ingress Secret: mhb-blockhead-broker +Workers: 1 +Worker Zones: sao01 +Version: 1.11.3_1524 +Owner: mbauer@us.ibm.com +Monitoring Dashboard: - +``` + +Set a region if necessary: + + bx cs region-set ap-north + +Look for *Ingress Subdomain*, ours is `mhb-blockhead-broker.sao01.containers.appdomain.cloud` + +```console +$ bx cs machine-types sng01 +OK +Name Cores Memory Network Speed OS Server Type Storage Secondary Storage Trustable +u2c.2x4 2 4GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +c2c.16x16 16 16GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +c2c.16x32 16 32GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +b2c.4x16 4 16GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +b2c.8x32 8 32GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +b2c.16x64 16 64GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +``` + + +look up the vlans to add + +``` +$ bx cs vlans --zone sng01 +OK +ID Name Number Type Router Supports Virtual Workers +2457919 1458 private bcr01a.sng01 true +2457917 1406 public fcr01a.sng01 true +``` + +``` +$ bx cs cluster-create --name peanuts --kube-version 1.11.3 --location sng01 --machine-type u2c.2x4 --workers 3 --public-vlan 2457917 --private-vlan 2457919 +Creating cluster... +OK +``` + +Eventually... + +``` +$ bx cs clusters +OK +Name ID State Created Workers Location Version +peanuts ed40c720877447e192fb51e09f7e4474 requested 30 seconds ago 1 sng01 1.11.3_1524 +``` diff --git a/k8s/deploy.yaml b/k8s/deploy.yaml new file mode 100644 index 00000000..70ecbfa4 --- /dev/null +++ b/k8s/deploy.yaml @@ -0,0 +1,62 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: blockhead-broker + namespace: default +data: + config.json: |+ + { + "username":"a", + "password":"b", + "port": 3333, + "deployer_path":"/pusher.js", + "container_manager": "kubernetes", + "external_address": "peanuts.sng01.containers.appdomain.cloud" + } +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: blockhead-broker + labels: + app: blockhead-broker +spec: + selector: + matchLabels: + app: blockhead-broker + replicas: 1 + template: + metadata: + labels: + app: blockhead-broker + spec: + containers: + - name: broker + image: mhbauer/blockhead-broker:eu2018 + command: ["/broker","/config/config.json","/services"] + ports: + - containerPort: 3333 + imagePullPolicy: Always + volumeMounts: + - name: config + mountPath: "/config" + volumes: + - name: config + configMap: + name: blockhead-broker +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: blockhead-broker + name: blockhead-broker + namespace: default +spec: + ports: + - port: 3333 + nodePort: 30000 + selector: + app: blockhead-broker + type: NodePort diff --git a/k8s/ingress.yaml b/k8s/ingress.yaml new file mode 100644 index 00000000..6a12319f --- /dev/null +++ b/k8s/ingress.yaml @@ -0,0 +1,16 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: blockhead-broker + namespace: default + annotations: + ingress.bluemix.net/rewrite-path: "serviceName=blockhead-broker rewrite=/" +spec: + rules: + - host: mhb-blockhead-broker.sao01.containers.appdomain.cloud + http: + paths: + - path: "/broker/" + backend: + serviceName: "blockhead-broker" + servicePort: 3333 diff --git a/k8s/n.yaml b/k8s/n.yaml new file mode 100644 index 00000000..6e5061a6 --- /dev/null +++ b/k8s/n.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + run: "n" + name: "n" +spec: + containers: + - image: nginx + name: "n" + ports: + - containerPort: 80 + restartPolicy: Never +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: "n" + name: "n" +spec: + ports: + - port: 8080 + targetPort: 80 + selector: + run: "n" + type: NodePort +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: "n" + namespace: default + annotations: + ingress.bluemix.net/rewrite-path: "serviceName=n rewrite=/" +spec: + rules: + - host: peanuts.sng01.containers.appdomain.cloud + http: + paths: + - path: "/nginx/" + backend: + serviceName: "n" + servicePort: 8080 diff --git a/k8s/rbac.yaml b/k8s/rbac.yaml new file mode 100644 index 00000000..13294a0b --- /dev/null +++ b/k8s/rbac.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: blockhead +rules: + - apiGroups: [""] + resources: ["pods","services"] + verbs: ["get","create","delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: default-blockhead +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: blockhead +subjects: +- kind: ServiceAccount + name: default + namespace: default + diff --git a/k8s/service-cm.yaml b/k8s/service-cm.yaml new file mode 100644 index 00000000..c03e3bd8 --- /dev/null +++ b/k8s/service-cm.yaml @@ -0,0 +1,26 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: blockhead-broker-services + namespace: default +data: + eth.json: |+ + { + "name": "keth", + "description": "Ethereum Geth Node", + "tags": [ + "eth", + "geth", + "dev" + ], + "display_name": "Geth 1.8", + "plans": [ + { + "name": "free", + "free": true, + "image": "mhbauer/blockhead-geth:eu2018", + "description": "Free Trial", + "ports": ["8545"] + } + ] + } diff --git a/pkg/containermanager/container_manager.go b/pkg/containermanager/container_manager.go index 237e634b..8e6c3243 100644 --- a/pkg/containermanager/container_manager.go +++ b/pkg/containermanager/container_manager.go @@ -19,10 +19,13 @@ type Binding struct { Port string } +// internal port to ip:externalport type ContainerInfo struct { ExternalAddress string + // used for internal communication of broker to container InternalAddress string - Bindings map[string][]Binding + // Bindings is a map of container ports to exposed host & port + Bindings map[string][]Binding } //go:generate counterfeiter -o ../fakes/fake_container_manager.go . ContainerManager diff --git a/pkg/containermanager/kubernetes/kubernetes_manager.go b/pkg/containermanager/kubernetes/kubernetes_manager.go index 296605ff..795b1061 100644 --- a/pkg/containermanager/kubernetes/kubernetes_manager.go +++ b/pkg/containermanager/kubernetes/kubernetes_manager.go @@ -3,33 +3,146 @@ package kubernetes import ( "context" "fmt" + "strconv" "code.cloudfoundry.org/lager" + "github.com/cloudfoundry-incubator/blockhead/pkg/containermanager" + + "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) type kubernetesContainerManager struct { client kubernetes.Interface + host string logger lager.Logger } -func NewKubernetesContainerManager(logger lager.Logger, client kubernetes.Interface) containermanager.ContainerManager { +func NewKubernetesContainerManager(logger lager.Logger, client kubernetes.Interface, host string) containermanager.ContainerManager { return kubernetesContainerManager{ client: client, + host: host, logger: logger.Session("kubernetes-container-manager"), } } func (kc kubernetesContainerManager) Provision(ctx context.Context, cc containermanager.ContainerConfig) error { - return fmt.Errorf("kube provision unimplemented") + var err error + selector := make(map[string]string) + selector["app"] = cc.Name + selector["provisionedBy"] = "blockhead-broker" + + blockheadNamespace := v1.NamespaceDefault // put everything in default? our own ns? + + // create a pod with a label + pod := &v1.Pod{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: cc.Name, + Namespace: blockheadNamespace, + Labels: selector, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + Name: cc.Name, + Image: cc.Image, + }, + }, + }, + } + + _, err = kc.client.CoreV1().Pods(v1.NamespaceDefault).Create(pod) + if err != nil { + kc.logger.Error("error creating pod", err) + } + + servicePorts := []v1.ServicePort{} + for _, p := range cc.ExposedPorts { + kc.logger.Debug("adding exposed port " + p) + port, err := strconv.Atoi(p) + if err != nil { + return err + } + sp := v1.ServicePort{ + Port: int32(port), + // we're not choosing a target port here. It will thus + // automatically be chosen when the service is created. + } + servicePorts = append(servicePorts, sp) + } + + // create a service with the same label and a label selector of the label + // nodeport for now? + svc := &v1.Service{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "B" + cc.Name, + Namespace: blockheadNamespace, + Labels: selector, + }, + Spec: v1.ServiceSpec{ + // ports, get from passed in + Ports: servicePorts, + Selector: selector, // selector use same as above + // service type is either nodeport or loadbalancer if we can + Type: v1.ServiceTypeNodePort, + }, + } + kc.logger.Debug(fmt.Sprintf("%++v\n", svc)) + _, err = kc.client.CoreV1().Services(v1.NamespaceDefault).Create(svc) + if err != nil { + kc.logger.Error("error creating service", err) + } + + return nil } func (kc kubernetesContainerManager) Deprovision(ctx context.Context, instanceID string) error { - return fmt.Errorf("kube deprovision unimplemented") + var err error + // TODO: investigate delete everything with a label selector + // everything uses the instance name so we don't have to track it in storage yet. + // ignore all the errors, because it doesn't hurt to fail to delete things. Log them instead. + err = kc.client.CoreV1().Pods(v1.NamespaceDefault).Delete(instanceID, &meta_v1.DeleteOptions{}) + if err != nil { + kc.logger.Info(err.Error()) + } + err = kc.client.CoreV1().Services(v1.NamespaceDefault).Delete("B" + instanceID, &meta_v1.DeleteOptions{}) + if err != nil { + kc.logger.Info(err.Error()) + } + return nil } func (kc kubernetesContainerManager) Bind(cts context.Context, bc containermanager.BindConfig) (*containermanager.ContainerInfo, error) { - kc.logger.Fatal("not-implemented", fmt.Errorf("not implemeneted")) - return nil, nil + + // pod and service both with instance name + instanceName := bc.InstanceId + + // take each port binding and find the nodeport that it is bound too. + instanceservice, err := kc.client.CoreV1().Services(v1.NamespaceDefault).Get("B" + instanceName, meta_v1.GetOptions{}) + if err != nil { + return nil, err + } + + ports := instanceservice.Spec.Ports + + bindings := make(map[string][]containermanager.Binding) + for _, port := range ports { + p := strconv.Itoa(int(port.NodePort)) + + containerBindings := []containermanager.Binding{} + containerBindings = append(containerBindings, containermanager.Binding{ + Port: p, + }) + bindings[strconv.Itoa(int(port.Port))] = containerBindings + } + + response := containermanager.ContainerInfo{ + ExternalAddress: kc.host, + InternalAddress: instanceName, + Bindings: bindings, + } + + return &response, nil } diff --git a/pkg/deployer/deployer.go b/pkg/deployer/deployer.go index 0bf9d7a7..3836af1c 100644 --- a/pkg/deployer/deployer.go +++ b/pkg/deployer/deployer.go @@ -58,7 +58,7 @@ func (e ethereumDeployer) DeployContract(contractInfo *ContractInfo, containerIn Password string `json:"password"` Args []string `json:"args"` }{ - Provider: fmt.Sprintf("http://%s:%s", containerInfo.InternalAddress, portBindings[0].Port), + Provider: fmt.Sprintf("http://%s:%s", containerInfo.InternalAddress, "8545"), Password: "", Args: contractInfo.ContractArgs, } @@ -74,6 +74,7 @@ func (e ethereumDeployer) DeployContract(contractInfo *ContractInfo, containerIn if err != nil { return nil, err } + e.logger.Debug(fmt.Sprintf("%++v", config)) outputFile, err := ioutil.TempFile("", uuid.New()) if err != nil { @@ -82,6 +83,7 @@ func (e ethereumDeployer) DeployContract(contractInfo *ContractInfo, containerIn defer os.RemoveAll(outputFile.Name()) cmd := exec.Command("node", e.deployerPath, "-c", configFile.Name(), "-o", outputFile.Name(), contractInfo.ContractPath) + e.logger.Debug(fmt.Sprintf("%++v", cmd)) output, err := cmd.CombinedOutput() if err != nil { e.logger.Error("run-failed", err, lager.Data{"output": string(output)}) diff --git a/vendor/k8s.io/client-go/README.md b/vendor/k8s.io/client-go/README.md index 4e5e4220..5a382ed5 100644 --- a/vendor/k8s.io/client-go/README.md +++ b/vendor/k8s.io/client-go/README.md @@ -1,4 +1,4 @@ -# client-go +re# client-go Go clients for talking to a [kubernetes](http://kubernetes.io/) cluster.