diff --git a/README.md b/README.md index a73ab06e..d2dbe495 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,8 @@ To do all the make steps: make + + +If both the BOSH release and the k8s release are going to be hooked up +to the same service-broker platform, `services/eth.json` needs to be +configured so that the class and plan identities do not collide. diff --git a/cmd/broker/main.go b/cmd/broker/main.go index 850afa14..6c4fe23b 100644 --- a/cmd/broker/main.go +++ b/cmd/broker/main.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "os" + "strconv" "code.cloudfoundry.org/lager" "github.com/docker/docker/client" @@ -55,7 +56,7 @@ func main() { if err != nil { logger.Fatal("could not set up a kubernetes-client", err) } - manager = kcm.NewKubernetesContainerManager(logger, clientset) + manager = kcm.NewKubernetesContainerManager(logger, clientset, state.Config.ExternalAddress) logger.Debug("using kubernetes containermanager") default: logger.Fatal("no container manager in config", fmt.Errorf("no container manager specified in config %q", configFilepath)) @@ -79,6 +80,6 @@ func main() { } http.Handle("/", log(brokerAPI)) - logger.Debug("listening on" + string(state.Config.Port)) + logger.Debug("listening on port:" + strconv.Itoa(int(state.Config.Port))) logger.Fatal("http-listen", http.ListenAndServe(fmt.Sprintf(":%d", state.Config.Port), nil)) } diff --git a/images/Dockerfile.broker b/images/Dockerfile.broker index f3258753..684e9258 100644 --- a/images/Dockerfile.broker +++ b/images/Dockerfile.broker @@ -1,7 +1,7 @@ from golang:1.11-alpine3.8 as builder workdir /go/src/github.com/cloudfoundry-incubator/blockhead copy . . -run go build -v -o /broker ./cmd/broker +run CGO_ENABLED=0 go build -ldflags '-s -w -d' -v -o /broker ./cmd/broker from node:10 diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 00000000..a0cd6119 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,159 @@ +This directory has the resources to install the broker in k8s. Only need to be +used once on startup. + +`rbac.yaml` sets up the default namespace so that the broker has the access +needed to create k8s objects. + +`deploy.yaml` installs the broker and creates a service to access it in-cluster. + +`ingress.yaml` sets up the basic ingress for accessing the broker from outside +the cluster by a resolving dns name. It needs to be edited to have the right +host as the ingress domain. + +`n.yaml` has test resources to troubleshoot ingress or other networking with a +very basic nginx setup. + +* How to K8s + +Prereqs: + - get a kube cluster + - know an address to access it (it's load-balancer or worker-node) + +RBAC setup. We only require access to create and delete services and pods. The +provided rbac has much more access because I am lazy. + +create it with : + + kubectl create -f rbac.yaml + +Edit the `deploy.yaml` to fixup the `external_address` field so that it points +at either the cluster loadbalancer or a publicly accessible node address (IP or DNS). +Now create the Broker, it's Service, and the config data necessary to start it with: + + kubectl create -f deploy.yaml + +Wait a bit and everything should be deployed. + + kubectl get all -l app=blockhead-broker + +In that should be the service and what nodeport it is bound to. + +We can get the catalog with curl. + +``` +curl -sSL a:b@peanuts.sng01.containers.appdomain.cloud:32089/v2/catalog -H +"X-Broker-API-Version: 2.14" + +{"services":[{"id":"0d25f970-899a-4aa4-b753-640e33b66389","name":"eth","description":"Ethereum +Geth +Node","bindable":true,"tags":["eth","geth","dev"],"plan_updateable":false,"plans":[{"id":"0a20f765-9e22-4ac2-b9f7-2ac8f706747c","name":"free","description":"Free +Trial","free":true}],"metadata":{"displayName":"Geth 1.8"}}]} +``` + +Do a service provision: + +```console +$ curl -sSL a:b@peanuts.sng01.containers.appdomain.cloud:30000/v2/service_instances/test-broker -d "@sp.json" -XPUT -H "X-Broker-API-Version: 2.14" -H "Content-Type: application/json" +``` +where `sp.json` contains the service and plan to provision. + + +Do a bind: + +```console +$ curl http://a:b@peanuts.sng01.containers.appdomain.cloud:30000/v2/service_instances/test-broker/service_bindings/newbind -d "@sp2.json" -XPUT -H "X-Broker-API-Version: 2.14" -H "Content-Type: application/json" +{"credentials":{"ContainerInfo":{"ExternalAddress":"peanuts.sng01.containers.appdomain.cloud","InternalAddress":"test-broker","Bindings":{"8545":[{"Port":"30971"}]}},"NodeInfo":{"address":"0x6361eFC13f0b5f0072e3774254a1e9a876db6BD7","abi":"[{\"constant\":false,\"inputs\":[],\"name\":\"voteA\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"proposalA\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[],\"name\":\"voteB\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"proposalB\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"name\":\"_\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"}]","contract_address":"0xA5948eDC459d0D3e83D6f9b9Cf089182EBE2b056","gas_price":"1","transaction_hash":"0xfa6d497809439d305cd64073de44c8e4b51ebc9d9a24fc1a651bd73a6f0b85c7"}}} +``` +Where `sp2.json` contains the service, plan, and parameters. + +If attaching remix from the webconsole, make sure that it's loaded from the insecure http backend. + +* cleanup tips + +Delete all the deployed geth nodes in one shot + + kubectl delete all -l provisionedBy=blockhead-broker + +Delete all the broker stuff in one shot + + kubectl delete all -l app=blockhead-broker + +* optional stuff + +using ingress is nice. Easiest to be used for the broker. The host role in `ingress.yaml` has to be adjusted. + + kubectl create -f ingress.yaml + +Can of course deploy the app in kubernetes. + + +* set up an IKS cluster + +On IKS, your loadbalancer DNS entry is found in the output of: + +```console +$ bx cs cluster-get mhb-blockhead-broker +Retrieving cluster mhb-blockhead-broker... +OK + + +Name: mhb-blockhead-broker +ID: 26760c5d3aca48619e4b79587feb1ce2 +State: normal +Created: 2018-09-29T01:47:10+0000 +Location: sao01 +Master URL: https://169.57.151.10:31423 +Master Location: sao01 +Master Status: Ready (2 days ago) +Ingress Subdomain: mhb-blockhead-broker.sao01.containers.appdomain.cloud +Ingress Secret: mhb-blockhead-broker +Workers: 1 +Worker Zones: sao01 +Version: 1.11.3_1524 +Owner: mbauer@us.ibm.com +Monitoring Dashboard: - +``` + +Set a region if necessary: + + bx cs region-set ap-north + +Look for *Ingress Subdomain*, ours is `mhb-blockhead-broker.sao01.containers.appdomain.cloud` + +```console +$ bx cs machine-types sng01 +OK +Name Cores Memory Network Speed OS Server Type Storage Secondary Storage Trustable +u2c.2x4 2 4GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +c2c.16x16 16 16GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +c2c.16x32 16 32GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +b2c.4x16 4 16GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +b2c.8x32 8 32GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +b2c.16x64 16 64GB 1000Mbps UBUNTU_16_64 virtual 25GB 100GB false +``` + + +look up the vlans to add + +``` +$ bx cs vlans --zone sng01 +OK +ID Name Number Type Router Supports Virtual Workers +2457919 1458 private bcr01a.sng01 true +2457917 1406 public fcr01a.sng01 true +``` + +``` +$ bx cs cluster-create --name peanuts --kube-version 1.11.3 --location sng01 --machine-type u2c.2x4 --workers 3 --public-vlan 2457917 --private-vlan 2457919 +Creating cluster... +OK +``` + +Eventually... + +``` +$ bx cs clusters +OK +Name ID State Created Workers Location Version +peanuts ed40c720877447e192fb51e09f7e4474 requested 30 seconds ago 1 sng01 1.11.3_1524 +``` diff --git a/k8s/deploy.yaml b/k8s/deploy.yaml new file mode 100644 index 00000000..70ecbfa4 --- /dev/null +++ b/k8s/deploy.yaml @@ -0,0 +1,62 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: blockhead-broker + namespace: default +data: + config.json: |+ + { + "username":"a", + "password":"b", + "port": 3333, + "deployer_path":"/pusher.js", + "container_manager": "kubernetes", + "external_address": "peanuts.sng01.containers.appdomain.cloud" + } +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: blockhead-broker + labels: + app: blockhead-broker +spec: + selector: + matchLabels: + app: blockhead-broker + replicas: 1 + template: + metadata: + labels: + app: blockhead-broker + spec: + containers: + - name: broker + image: mhbauer/blockhead-broker:eu2018 + command: ["/broker","/config/config.json","/services"] + ports: + - containerPort: 3333 + imagePullPolicy: Always + volumeMounts: + - name: config + mountPath: "/config" + volumes: + - name: config + configMap: + name: blockhead-broker +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: blockhead-broker + name: blockhead-broker + namespace: default +spec: + ports: + - port: 3333 + nodePort: 30000 + selector: + app: blockhead-broker + type: NodePort diff --git a/k8s/ingress.yaml b/k8s/ingress.yaml new file mode 100644 index 00000000..6a12319f --- /dev/null +++ b/k8s/ingress.yaml @@ -0,0 +1,16 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: blockhead-broker + namespace: default + annotations: + ingress.bluemix.net/rewrite-path: "serviceName=blockhead-broker rewrite=/" +spec: + rules: + - host: mhb-blockhead-broker.sao01.containers.appdomain.cloud + http: + paths: + - path: "/broker/" + backend: + serviceName: "blockhead-broker" + servicePort: 3333 diff --git a/k8s/n.yaml b/k8s/n.yaml new file mode 100644 index 00000000..6e5061a6 --- /dev/null +++ b/k8s/n.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + run: "n" + name: "n" +spec: + containers: + - image: nginx + name: "n" + ports: + - containerPort: 80 + restartPolicy: Never +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: "n" + name: "n" +spec: + ports: + - port: 8080 + targetPort: 80 + selector: + run: "n" + type: NodePort +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: "n" + namespace: default + annotations: + ingress.bluemix.net/rewrite-path: "serviceName=n rewrite=/" +spec: + rules: + - host: peanuts.sng01.containers.appdomain.cloud + http: + paths: + - path: "/nginx/" + backend: + serviceName: "n" + servicePort: 8080 diff --git a/k8s/rbac.yaml b/k8s/rbac.yaml new file mode 100644 index 00000000..13294a0b --- /dev/null +++ b/k8s/rbac.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: blockhead +rules: + - apiGroups: [""] + resources: ["pods","services"] + verbs: ["get","create","delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: default-blockhead +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: blockhead +subjects: +- kind: ServiceAccount + name: default + namespace: default + diff --git a/k8s/service-cm.yaml b/k8s/service-cm.yaml new file mode 100644 index 00000000..c03e3bd8 --- /dev/null +++ b/k8s/service-cm.yaml @@ -0,0 +1,26 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: blockhead-broker-services + namespace: default +data: + eth.json: |+ + { + "name": "keth", + "description": "Ethereum Geth Node", + "tags": [ + "eth", + "geth", + "dev" + ], + "display_name": "Geth 1.8", + "plans": [ + { + "name": "free", + "free": true, + "image": "mhbauer/blockhead-geth:eu2018", + "description": "Free Trial", + "ports": ["8545"] + } + ] + } diff --git a/pkg/containermanager/container_manager.go b/pkg/containermanager/container_manager.go index 237e634b..8e6c3243 100644 --- a/pkg/containermanager/container_manager.go +++ b/pkg/containermanager/container_manager.go @@ -19,10 +19,13 @@ type Binding struct { Port string } +// internal port to ip:externalport type ContainerInfo struct { ExternalAddress string + // used for internal communication of broker to container InternalAddress string - Bindings map[string][]Binding + // Bindings is a map of container ports to exposed host & port + Bindings map[string][]Binding } //go:generate counterfeiter -o ../fakes/fake_container_manager.go . ContainerManager diff --git a/pkg/containermanager/kubernetes/kubernetes_manager.go b/pkg/containermanager/kubernetes/kubernetes_manager.go index 296605ff..795b1061 100644 --- a/pkg/containermanager/kubernetes/kubernetes_manager.go +++ b/pkg/containermanager/kubernetes/kubernetes_manager.go @@ -3,33 +3,146 @@ package kubernetes import ( "context" "fmt" + "strconv" "code.cloudfoundry.org/lager" + "github.com/cloudfoundry-incubator/blockhead/pkg/containermanager" + + "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) type kubernetesContainerManager struct { client kubernetes.Interface + host string logger lager.Logger } -func NewKubernetesContainerManager(logger lager.Logger, client kubernetes.Interface) containermanager.ContainerManager { +func NewKubernetesContainerManager(logger lager.Logger, client kubernetes.Interface, host string) containermanager.ContainerManager { return kubernetesContainerManager{ client: client, + host: host, logger: logger.Session("kubernetes-container-manager"), } } func (kc kubernetesContainerManager) Provision(ctx context.Context, cc containermanager.ContainerConfig) error { - return fmt.Errorf("kube provision unimplemented") + var err error + selector := make(map[string]string) + selector["app"] = cc.Name + selector["provisionedBy"] = "blockhead-broker" + + blockheadNamespace := v1.NamespaceDefault // put everything in default? our own ns? + + // create a pod with a label + pod := &v1.Pod{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: cc.Name, + Namespace: blockheadNamespace, + Labels: selector, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + Name: cc.Name, + Image: cc.Image, + }, + }, + }, + } + + _, err = kc.client.CoreV1().Pods(v1.NamespaceDefault).Create(pod) + if err != nil { + kc.logger.Error("error creating pod", err) + } + + servicePorts := []v1.ServicePort{} + for _, p := range cc.ExposedPorts { + kc.logger.Debug("adding exposed port " + p) + port, err := strconv.Atoi(p) + if err != nil { + return err + } + sp := v1.ServicePort{ + Port: int32(port), + // we're not choosing a target port here. It will thus + // automatically be chosen when the service is created. + } + servicePorts = append(servicePorts, sp) + } + + // create a service with the same label and a label selector of the label + // nodeport for now? + svc := &v1.Service{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "B" + cc.Name, + Namespace: blockheadNamespace, + Labels: selector, + }, + Spec: v1.ServiceSpec{ + // ports, get from passed in + Ports: servicePorts, + Selector: selector, // selector use same as above + // service type is either nodeport or loadbalancer if we can + Type: v1.ServiceTypeNodePort, + }, + } + kc.logger.Debug(fmt.Sprintf("%++v\n", svc)) + _, err = kc.client.CoreV1().Services(v1.NamespaceDefault).Create(svc) + if err != nil { + kc.logger.Error("error creating service", err) + } + + return nil } func (kc kubernetesContainerManager) Deprovision(ctx context.Context, instanceID string) error { - return fmt.Errorf("kube deprovision unimplemented") + var err error + // TODO: investigate delete everything with a label selector + // everything uses the instance name so we don't have to track it in storage yet. + // ignore all the errors, because it doesn't hurt to fail to delete things. Log them instead. + err = kc.client.CoreV1().Pods(v1.NamespaceDefault).Delete(instanceID, &meta_v1.DeleteOptions{}) + if err != nil { + kc.logger.Info(err.Error()) + } + err = kc.client.CoreV1().Services(v1.NamespaceDefault).Delete("B" + instanceID, &meta_v1.DeleteOptions{}) + if err != nil { + kc.logger.Info(err.Error()) + } + return nil } func (kc kubernetesContainerManager) Bind(cts context.Context, bc containermanager.BindConfig) (*containermanager.ContainerInfo, error) { - kc.logger.Fatal("not-implemented", fmt.Errorf("not implemeneted")) - return nil, nil + + // pod and service both with instance name + instanceName := bc.InstanceId + + // take each port binding and find the nodeport that it is bound too. + instanceservice, err := kc.client.CoreV1().Services(v1.NamespaceDefault).Get("B" + instanceName, meta_v1.GetOptions{}) + if err != nil { + return nil, err + } + + ports := instanceservice.Spec.Ports + + bindings := make(map[string][]containermanager.Binding) + for _, port := range ports { + p := strconv.Itoa(int(port.NodePort)) + + containerBindings := []containermanager.Binding{} + containerBindings = append(containerBindings, containermanager.Binding{ + Port: p, + }) + bindings[strconv.Itoa(int(port.Port))] = containerBindings + } + + response := containermanager.ContainerInfo{ + ExternalAddress: kc.host, + InternalAddress: instanceName, + Bindings: bindings, + } + + return &response, nil } diff --git a/pkg/deployer/deployer.go b/pkg/deployer/deployer.go index 0bf9d7a7..3836af1c 100644 --- a/pkg/deployer/deployer.go +++ b/pkg/deployer/deployer.go @@ -58,7 +58,7 @@ func (e ethereumDeployer) DeployContract(contractInfo *ContractInfo, containerIn Password string `json:"password"` Args []string `json:"args"` }{ - Provider: fmt.Sprintf("http://%s:%s", containerInfo.InternalAddress, portBindings[0].Port), + Provider: fmt.Sprintf("http://%s:%s", containerInfo.InternalAddress, "8545"), Password: "", Args: contractInfo.ContractArgs, } @@ -74,6 +74,7 @@ func (e ethereumDeployer) DeployContract(contractInfo *ContractInfo, containerIn if err != nil { return nil, err } + e.logger.Debug(fmt.Sprintf("%++v", config)) outputFile, err := ioutil.TempFile("", uuid.New()) if err != nil { @@ -82,6 +83,7 @@ func (e ethereumDeployer) DeployContract(contractInfo *ContractInfo, containerIn defer os.RemoveAll(outputFile.Name()) cmd := exec.Command("node", e.deployerPath, "-c", configFile.Name(), "-o", outputFile.Name(), contractInfo.ContractPath) + e.logger.Debug(fmt.Sprintf("%++v", cmd)) output, err := cmd.CombinedOutput() if err != nil { e.logger.Error("run-failed", err, lager.Data{"output": string(output)}) diff --git a/vendor/k8s.io/client-go/README.md b/vendor/k8s.io/client-go/README.md index 4e5e4220..5a382ed5 100644 --- a/vendor/k8s.io/client-go/README.md +++ b/vendor/k8s.io/client-go/README.md @@ -1,4 +1,4 @@ -# client-go +re# client-go Go clients for talking to a [kubernetes](http://kubernetes.io/) cluster.