From eadb04aff95a383a60b61f83dc6a8ee02d1370cc Mon Sep 17 00:00:00 2001 From: Antoine Legrand Date: Fri, 28 Apr 2017 21:26:56 +0200 Subject: [PATCH 1/5] Add kube.libsonnet --- .style.yapf | 3 +- MANIFEST.in | 5 +- kpm/jsonnet/lib/core.libsonnet | 976 ++++++++++++++++++ kpm/jsonnet/lib/internal/assert.libsonnet | 25 + kpm/jsonnet/lib/internal/base.libsonnet | 15 + kpm/jsonnet/lib/internal/meta.libsonnet | 11 + ...m-utils.libjsonnet => kpm-utils.libsonnet} | 0 .../lib/{kpm.libjsonnet => kpm.libsonnet} | 6 +- kpm/jsonnet/lib/kube.libsonnet | 1 + kpm/jsonnet/lib/util.libsonnet | 90 ++ kpm/jsonnet/manifest.jsonnet.j2 | 2 +- kpm/render_jsonnet.py | 32 +- 12 files changed, 1143 insertions(+), 23 deletions(-) create mode 100644 kpm/jsonnet/lib/core.libsonnet create mode 100644 kpm/jsonnet/lib/internal/assert.libsonnet create mode 100644 kpm/jsonnet/lib/internal/base.libsonnet create mode 100644 kpm/jsonnet/lib/internal/meta.libsonnet rename kpm/jsonnet/lib/{kpm-utils.libjsonnet => kpm-utils.libsonnet} (100%) rename kpm/jsonnet/lib/{kpm.libjsonnet => kpm.libsonnet} (96%) create mode 160000 kpm/jsonnet/lib/kube.libsonnet create mode 100644 kpm/jsonnet/lib/util.libsonnet diff --git a/.style.yapf b/.style.yapf index 3456981..13a9dea 100644 --- a/.style.yapf +++ b/.style.yapf @@ -11,8 +11,7 @@ ALLOW_MULTILINE_DICTIONARY_KEYS=True # False BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=True BLANK_LINE_BEFORE_CLASS_DOCSTRING=False -# False -COALESCE_BRACKETS=False +COALESCE_BRACKETS=True CONTINUATION_INDENT_WIDTH=4 DEDENT_CLOSING_BRACKETS=False diff --git a/MANIFEST.in b/MANIFEST.in index 17e0d81..bc4a0ca 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,10 +4,7 @@ include LICENSE include README.md include requirements.txt include requirements_dev.txt -include kpm/jsonnet/*.jsonnet -include kpm/jsonnet/*.libjsonnet -include kpm/jsonnet/lib/*.libjsonnet -include kpm/jsonnet/manifest.jsonnet.j2 +recursive-include kpm/jsonnet * recursive-include tests * recursive-exclude * __pycache__ recursive-exclude * *.py[co] diff --git a/kpm/jsonnet/lib/core.libsonnet b/kpm/jsonnet/lib/core.libsonnet new file mode 100644 index 0000000..5420042 --- /dev/null +++ b/kpm/jsonnet/lib/core.libsonnet @@ -0,0 +1,976 @@ +local kubeAssert = import "internal/assert.libsonnet"; +local base = import "internal/base.libsonnet"; +local meta = import "internal/meta.libsonnet"; + +{ + // A collection of common fields in the Kubernetes API objects, + // that we do not want to expose for public use. For example, + // `Kind` appears frequently in API objects of both + // `extensions/v1beta1` and `v1`, but we don't want users to mess + // mess with an object's `Kind`. + local common = { + Kind(kind):: kubeAssert.Type("kind", kind, "string") {kind: kind}, + + // TODO: This sets the metadata property, rather than doing a + // mixin. Is this what we want? + Metadata(metadata={}):: {metadata: $.v1.metadata.Default() + metadata}, + + mixin:: { + Metadata(mixin):: {metadata+: mixin}, + + metadata:: { + local metadata = $.v1.metadata, + local mixin = common.mixin, + + Name:: meta.MixinPartial1(metadata.Name, mixin.Metadata), + Label:: meta.MixinPartial2(metadata.Label, mixin.Metadata), + Labels:: meta.MixinPartial1(metadata.Labels, mixin.Metadata), + Namespace:: meta.MixinPartial1(metadata.Namespace, mixin.Metadata), + Annotation:: meta.MixinPartial2(metadata.Annotation, mixin.Metadata), + Annotations:: + meta.MixinPartial1(metadata.Annotations, mixin.Metadata), + }, + }, + }, + + v1:: { + local bases = { + ConfigMap: base.New("configMap", "AC74E727-0605-4872-8F30-E5CAFB2A0984"), + Container: base.New("container", "50281784-097C-46A9-8D2C-C6E9078D77D4"), + ContainerPort: + base.New("containerPort", "2854EB13-644C-4FEF-A62D-DBAC554D6A24"), + Metadata: base.New("metadata", "027AE69D-1DD6-42D2-AD47-8F4A55DF9D76"), + PersistentVolume: + base.New("persistentVolume", "03113473-7083-4D07-A7FE-83699EB4128C"), + PersistentVolumeClaim: + base.New("persistentVolumeClaim", "CD58B997-FF5E-4ED9-8F8A-573E92336D35"), + Pod: base.New("pod", "2854EB13-644C-4FEF-A62D-DBAC554D6A24"), + Probe: base.New("probe", "943CF775-B17F-4D25-A794-7D800F08E7FE"), + Secret: base.New("secret", "0C3D2362-968B-4751-BF67-D58ADA1FC5FC"), + Service: base.New("service", "87EE499C-EC06-421D-9450-EFE0701851EB"), + ServicePort: base.New("servicePort", "C38839B7-DA05-4845-B643-E6826E38EA1B"), + Mount: base.New("mount", "D1E2E601-E64A-4A95-A15C-E78CA724764C"), + Namespace: base.New("namespace", "6A94A118-F6A7-40EE-8BA1-6096CEC7BDE3"), + }, + + ApiVersion:: { apiVersion: "v1" }, + + metadata:: { + Default(name=null, namespace=null, annotations=null, labels=null):: + bases.Metadata + + (if name != null then self.Name(name) else {}) + + (if namespace != null then self.Namespace(namespace) else {}) + { + annotations: if annotations == null then {} else annotations, + labels: if labels == null then {} else labels, + }, + + Name(name):: + base.Verify(bases.Metadata) + + kubeAssert.Type("name", name, "string") + + {name: name}, + + Label(key, value):: + base.Verify(bases.Metadata) + + {labels+: {[key]: value}}, + + Labels(labels):: + base.Verify(bases.Metadata) + + {labels+: labels}, + + Namespace(namespace):: + base.Verify(bases.Metadata) + + kubeAssert.Type("namespace", namespace, "string") + + {namespace: namespace}, + + Annotation(key, value):: + base.Verify(bases.Metadata) + + {annotations+: {[key]: value}}, + + Annotations(annotations):: + base.Verify(bases.Metadata) + + {annotations+: annotations}, + }, + + // + // Namespace. + // + + namespace:: { + Default(name):: + bases.Namespace + + kubeAssert.Type("name", name, "string") + + $.v1.ApiVersion + + common.Kind("Namespace") + + common.Metadata($.v1.metadata.Name(name)), + }, + + // + // Ports. + // + port:: { + local protocolOptions = std.set(["TCP", "UDP"]), + + local PortProtocol(protocol, targetBase) = + kubeAssert.InSet("protocol", protocol, protocolOptions) + + base.Verify(targetBase) { + protocol: protocol, + }, + + local PortName(name, targetPort) = + base.Verify(targetPort) + + kubeAssert.Type("name", name, "string") { + name: name, + }, + + container:: { + Default(containerPort):: + bases.ContainerPort + + kubeAssert.ValidPort("containerPort", containerPort) { + containerPort: containerPort, + }, + + Named(name, containerPort):: + kubeAssert.Type("name", name, "string") + + self.Default(containerPort) + + self.Name(name), + + Name(name):: PortName(name, bases.ContainerPort), + + Protocol(protocol):: PortProtocol(protocol, bases.ContainerPort), + + HostPort(hostPort):: + base.Verify(bases.ContainerPort) + + kubeAssert.ValidPort("hostPort", hostPort) { + hostPort: hostPort + }, + + HostIp(hostIp):: + base.Verify(bases.ContainerPort) + + kubeAssert.Type("hostIp", hostIp, "string") { + hostIP: hostIp, + }, + }, + + service:: { + Default(servicePort):: + bases.ServicePort + + kubeAssert.ValidPort("servicePort", servicePort) { + port: servicePort, + }, + + WithTarget(servicePort, targetPort):: + self.Default(servicePort) + + self.TargetPort(targetPort), + + Named(name, servicePort, targetPort):: + kubeAssert.Type("name", name, "string") + + self.Default(servicePort) + + self.Name(name) + + self.TargetPort(targetPort), + + Name(name):: PortName(name, bases.ServicePort), + + Protocol(protocol):: PortProtocol(protocol, bases.ServicePort), + + TargetPort(targetPort):: + base.Verify(bases.ServicePort) { + // TODO: Assert clusterIP is not set? + targetPort: targetPort, + }, + + NodePort(nodePort):: + base.Verify(bases.ServicePort) { + nodePort: nodePort, + }, + }, + }, + + // + // Service. + // + + service:: { + Default(name, portList, labels={}, annotations={}):: + local defaultMetadata = + common.Metadata( + $.v1.metadata.Name(name) + + $.v1.metadata.Labels(labels) + + $.v1.metadata.Annotations(annotations)); + local serviceKind = common.Kind("Service"); + bases.Service + $.v1.ApiVersion + serviceKind + defaultMetadata { + spec: { + ports: portList, + }, + }, + + // TODO: Incorrect indentation below. + Metadata:: common.mixin.Metadata, + Spec(mixin):: {spec+: mixin}, + + spec:: { + local typeOptions = std.set([ + "ExternalName", "ClusterIP", "NodePort", "LoadBalancer"]), + local sessionAffinityOptions = std.set(["ClientIP", "None"]), + + Port(port):: + // base.Verify(bases.Service) + + {ports+: [port]}, + + Selector(selector):: + // base.Verify(bases.Service) + + {selector: selector}, + + ClusterIp(clusterIp):: + // base.Verify(bases.Service) + + kubeAssert.Type("clusterIp", clusterIp, "string") + + {clusterIP: clusterIp}, + + Type(type):: + // base.Verify(bases.Service) + + kubeAssert.InSet("type", type, typeOptions) + + {type: type}, + + ExternalIps(externalIpList):: + // base.Verify(bases.Service) + + // TODO: Verify that externalIpList is a list of string. + kubeAssert.Type("externalIpList", externalIpList, "array") + + {externalIPs: externalIpList}, + + SessionAffinity(sessionAffinity):: + // base.Verify(bases.Service) + + kubeAssert.InSet( + "sessionAffinity", sessionAffinity, sessionAffinityOptions) + + {sessionAffinity: sessionAffinity}, + + LoadBalancerIp(loadBalancerIp):: + // base.Verify(bases.Service) + + kubeAssert.Type("loadBalancerIp", loadBalancerIp, "string") + + {loadBalancerIP: loadBalancerIp}, + + LoadBalancerSourceRanges(loadBalancerSourceRanges):: + // base.Verify(bases.Service) + + // TODO: Verify that loadBalancerSourceRanges is a list + // of string. + kubeAssert.Type( + "loadBalancerSourceRanges", loadBalancerSourceRanges, "array") + + {loadBalancerSourceRanges: loadBalancerSourceRanges}, + + ExternalName(externalName):: + // base.Verify(bases.Service) + + kubeAssert.Type("externalName", externalName, "string") + + {externalName: externalName}, + }, + + mixin:: { + metadata:: common.mixin.metadata { + annotation:: { + TolerateUnreadyEndpoints(truthiness):: + common.mixin.metadata.Annotation( + "service.alpha.kubernetes.io/tolerate-unready-endpoints", + truthiness), + }, + }, + + spec:: { + local service = $.v1.service, + + Port:: + meta.MixinPartial1(service.spec.Port, service.Spec), + Selector:: + meta.MixinPartial1(service.spec.Selector, service.Spec), + ClusterIp:: + meta.MixinPartial1(service.spec.ClusterIp, service.Spec), + Type:: + meta.MixinPartial1(service.spec.Type, service.Spec), + ExternalIps:: + meta.MixinPartial1(service.spec.ExternalIps, service.Spec), + SessionAffinity:: + meta.MixinPartial1(service.spec.SessionAffinity, service.Spec), + LoadBalancerIp:: + meta.MixinPartial1(service.spec.LoadBalancerIp, service.Spec), + LoadBalancerSourceRanges:: meta.MixinPartial1( + service.spec.LoadBalancerSourceRanges, service.Spec), + ExternalName:: + meta.MixinPartial1(service.spec.ExternalName, service.Spec), + }, + }, + }, + + configMap:: { + Default(namespace, configMapName, data): + bases.ConfigMap + + $.v1.ApiVersion + + common.Kind("ConfigMap") + + common.Metadata( + $.v1.metadata.Name(configMapName) + + $.v1.metadata.Namespace(namespace)) { + data: data, + }, + + DefaultFromClaim(namespace, name, claim):: + self.Default(namespace, name, claim.metadata.name) + }, + + secret:: { + Default(namespace, secretName, data):: + bases.Secret + + $.v1.ApiVersion + + common.Kind("Secret") + + common.Metadata( + $.v1.metadata.Name(secretName) + + $.v1.metadata.Namespace(namespace)) { + data: data, + }, + + StringData(stringData):: + base.Verify(bases.Secret) { + stringData: stringData, + }, + + Type(type):: + base.Verify(bases.Secret) + + kubeAssert.Type("type", type, "string") { + type: type, + }, + }, + + // + // Volume. + // + + // + // NOTE: TODO: YOU ARE HERE. You haven't implemented type checking + // beyond this point. + // + + volume:: { + persistent:: { + // TODO: Add checks to the parameters here. + Default(name, claimName):: bases.PersistentVolume { + name: name, + persistentVolumeClaim: { + claimName: claimName, + }, + }, + + DefaultFromClaim(name, claim):: + self.Default(name, claim.metadata.name) + }, + + hostPath:: { + // TODO: Add checks to the parameters here. + Default(name, path):: { + name: name, + hostPath: { + path: path + }, + }, + }, + + configMap:: { + // TODO: Add checks to the parameters here. + Default(name, configMapName):: { + name: name, + configMap: { + name: configMapName, + }, + }, + }, + + secret:: { + // TODO: Add checks to the parameters here. + Default(name, secretName):: { + name: name, + secret: { + secretName: secretName, + }, + }, + }, + + emptyDir:: { + Default(name):: { + name: name, + emptyDir: {}, + }, + }, + + // + // Mount. + // + mount:: { + Default(name, mountPath, readOnly=false):: bases.Mount { + name: name, + mountPath: mountPath, + readOnly: readOnly, + }, + + FromVolume(volume, mountPath, readOnly=false):: + self.Default(volume.name, mountPath, readOnly), + + FromConfigMap(configMap, mountPath, readOnly=false):: + self.Default(configMap.name, mountPath, readOnly), + }, + + // + // Claim. + // + claim:: { + DefaultPersistent(claimName, accessModes, size, namespace=null): + local defaultMetadata = common.Metadata( + $.v1.metadata.Default(namespace=namespace, name=claimName)); + bases.PersistentVolumeClaim + + $.v1.ApiVersion + + common.Kind("PersistentVolumeClaim") + + defaultMetadata { + // TODO: Move this assert to `kubeAssert.Type`. + assert std.type(accessModes) == "array" + : "'accessModes' must by of type 'array'", + spec: { + accessModes: accessModes, + resources: { + requests: { + storage: size + }, + }, + }, + }, + + mixin:: { + metadata:: common.mixin.metadata { + annotation:: { + AlphaStorageClass(storageClass):: + common.mixin.metadata.Annotation( + "volume.alpha.kubernetes.io/storage-class", + storageClass), + + BetaStorageClass(storageClass):: + common.mixin.metadata.Annotation( + "volume.beta.kubernetes.io/storage-class", + storageClass), + }, + }, + }, + }, + }, + + // + // Probe. + // + probe:: { + local defaultTimeout = 1, + local defaultPeriod = 10, + + Default( + initDelaySecs, + timeoutSecs=defaultTimeout, + periodSeconds=defaultPeriod + ):: bases.Probe { + initialDelaySeconds: initDelaySecs, + timeoutSeconds: timeoutSecs, + }, + + Http( + getPath, + portName, + initDelaySecs, + timeoutSecs=defaultTimeout, + periodSeconds=defaultPeriod + ):: self.Default(initDelaySecs, timeoutSecs) { + httpGet: { + path: getPath, + port: portName, + }, + }, + + Tcp( + port, + initDelaySecs, + timeoutSecs=defaultTimeout, + periodSeconds=defaultPeriod + ):: self.Default(initDelaySecs, timeoutSecs) { + tcpSocket: { + port: port, + }, + }, + + Exec( + command, + initDelaySecs, + timeoutSecs=defaultTimeout, + periodSeconds=defaultPeriod + ):: self.Default(initDelaySecs, timeoutSecs) { + exec: { + command: command, + }, + }, + }, + + // + // Container. + // + container:: { + local imagePullPolicyOptions = std.set(["Always", "Never", "IfNotPresent"]), + + Default(name, image, imagePullPolicy="Always"):: + bases.Container + + // TODO: Make "Always" the default only when we're doing the :latest. + kubeAssert.Type("name", name, "string") + + kubeAssert.Type("image", image, "string") + + kubeAssert.InSet("imagePullPolicy", imagePullPolicy, imagePullPolicyOptions) { + name: name, + image: image, + imagePullPolicy: imagePullPolicy, + // TODO: Think carefully about whether we want an empty list here. + ports: [], + env: [], + volumeMounts: [], + }, + + Args(args):: base.Verify(bases.Container) { + args: args + }, + + Command(command):: base.Verify(bases.Container) { + command: command, + }, + + // TODO: Should this take a k/v pair instead? + Env(env):: base.Verify(bases.Container) { + env+: env, + }, + + Resources(resources):: base.Verify(bases.Container) { + resources: resources + }, + + Ports(ports):: base.Verify(bases.Container) { + ports+: ports, + }, + + Port(port):: base.Verify(bases.Container) { ports+: [port] }, + + NamedPort(name, port):: base.Verify(bases.Container) { + ports+: [$.v1.port.container.Named(name, port)], + }, + + LivenessProbe(probe):: base.Verify(bases.Container) { + livenessProbe: probe, + }, + + ReadinessProbe(probe):: base.Verify(bases.Container) { + readinessProbe: probe, + }, + + VolumeMounts(mounts):: base.Verify(bases.Container) { + volumeMounts+: mounts, + }, + + // TODO: Make these into mixins, also. + resources:: { + Requests(cpu, memory):: { + requests: { + cpu: cpu, + memory: memory + }, + }, + + Limits(cpu, memory):: { + limits: { + cpu: cpu, + memory: memory + }, + }, + }, + }, + + // + // Env. + // + env:: { + Variable(name, value):: { + name: name, + value: value, + }, + + ValueFrom(name, configMapName, configMapKey):: { + name: name, + valueFrom: { + configMapKeyRef: { + name: configMapName, + key: configMapKey, + }, + }, + }, + + ValueFromSecret(name, secretName, secretKey):: { + name: name, + valueFrom: { + secretKeyRef: { + name: secretName, + key: secretKey, + }, + }, + }, + }, + + // + // Pods. + // + pod:: { + local pod = self, + + Default(containers, volumes=[]):: + bases.Pod + + $.v1.ApiVersion + + common.Kind("Pod") + + common.Metadata() { + spec: pod.spec.Default(containers, volumes), + }, + + Metadata:: common.mixin.Metadata, + Spec(mixin):: {spec+: mixin}, + + spec:: { + Default(containers, volumes=[]):: { + containers: containers, + volumes: volumes, + }, + + // TODO: Consider making this a mixin. + Volumes(volumes):: {volumes: volumes}, + Containers(containers):: {containers: containers}, + DnsPolicy:: CreateDnsPolicyFunction(), + RestartPolicy:: CreateRestartPolicyFunction(), + }, + + template:: { + Default(containers, volumes=[]):: + common.Metadata() { + spec: pod.spec.Default(containers, volumes), + }, + + Metadata:: common.mixin.Metadata, + + mixin:: { + metadata:: common.mixin.metadata { + annotation:: { + PodAffinity(affinitySpec):: + common.mixin.metadata.Annotation( + "scheduler.alpha.kubernetes.io/affinity", affinitySpec), + PodInitContainers(initSpec):: + common.mixin.metadata.Annotation( + "pod.alpha.kubernetes.io/init-containers", initSpec), + }, + }, + + spec:: { + local pod = $.v1.pod, + local templateSpecMixin(mixin) = {template+: {spec+: mixin}}, + + Containers:: + meta.MixinPartial1(pod.spec.Containers, templateSpecMixin), + Volumes:: + meta.MixinPartial1(pod.spec.Volumes, templateSpecMixin), + DnsPolicy:: + meta.MixinPartial1(pod.spec.DnsPolicy, templateSpecMixin), + RestartPolicy:: + meta.MixinPartial1(pod.spec.RestartPolicy, templateSpecMixin), + }, + }, + }, + + local CreateDnsPolicyFunction(createMixin=null) = + local partial = meta.MixinPartial1( + function(policy) {dnsPolicy: policy}, + createMixin); + function(policy="ClusterFirst") partial(policy), + + local CreateRestartPolicyFunction(createMixin=null) = + local partial = meta.MixinPartial1( + function(policy) {restartPolicy: policy}, + createMixin); + function(policy="Always") partial(policy), + }, + }, + + extensions:: { + v1beta1: { + local bases = { + Deployment: base.New("deployment", "176A7BEF-E577-4EBD-952D-5E8F7BB7AE1A"), + }, + + ApiVersion:: { apiVersion: "extensions/v1beta1" }, + + // + // Deployments. + // + deployment:: { + // TODO: Get rid of the `spec` parameter. + Default(name, spec):: + bases.Deployment + + $.extensions.v1beta1.ApiVersion + + common.Kind("Deployment") + + common.Metadata($.v1.metadata.Name(name)) { + spec: spec, + }, + + Metadata:: common.mixin.Metadata, + Spec(mixin):: {spec+: mixin}, + + spec:: { + ReplicatedPod(replicas, podTemplate):: { + replicas: replicas, + // TODO: Should this be a mixin? + template: podTemplate, + }, + + NodeSelector(labels):: + // base.Verify(bases.Service) + + {nodeSelector: labels}, + + Selector(labels):: { + // base.Verify(bases.Service) + + // TODO: Consider making these mixins. + selector: { + matchLabels: labels, + }, + }, + + MinReadySeconds:: CreateMinReadySecondsFunction(), + RollingUpdateStrategy:: CreateRollingUpdateStrategyFunction(), + }, + + mixin:: { + metadata:: common.mixin.metadata, + + podTemplate:: { + local pod = $.v1.pod, + + Volumes:: meta.MixinPartial1(pod.spec.Volumes, podMixin), + Containers:: meta.MixinPartial1(pod.spec.Containers, podMixin), + + // TODO: Consider moving this default to some common + // place, so it's not duplicated. + DnsPolicy:: + local partial = + meta.MixinPartial1(pod.spec.DnsPolicy, podMixin); + function(policy="ClusterFirst") partial(policy), + + RestartPolicy(policy="Always"):: + podMixin(pod.spec.RestartPolicy(policy=policy)), + + local podMixin(mixin) = { + // TODO: Add base verification here. + spec+: { + template+: { + spec+: mixin + }, + }, + }, + }, + + spec:: { + local deployment = $.extensions.v1beta1.deployment, + + NodeSelector:: meta.MixinPartial1( + deployment.spec.NodeSelector, deployment.Spec), + Selector:: + meta.MixinPartial1(deployment.spec.Selector, deployment.Spec), + MinReadySeconds:: CreateMinReadySecondsFunction(deployment.Spec), + RollingUpdateStrategy:: + CreateRollingUpdateStrategyFunction(deployment.Spec), + }, + }, + + local CreateMinReadySecondsFunction(createMixin=null) = + local partial = + meta.MixinPartial1( + function(seconds) + // base.Verify(bases.Service) + + {minReadySeconds: seconds}, + createMixin); + function(seconds=0) partial(seconds), + + local CreateRollingUpdateStrategyFunction(createMixin=null) = + local rollingUpdateStrategy(maxSurge, maxUnavailable) = { + // base.Verify(bases.Service) + strategy: { + rollingUpdate: { + maxSurge: maxSurge, + maxUnavailable: maxUnavailable, + }, + type: "RollingUpdate", + }, + }; + local partial = + meta.MixinPartial2( + rollingUpdateStrategy, + createMixin); + function(maxSurge=1, maxUnavailable=1) + partial(maxSurge, maxUnavailable), + }, + + ingress:: { + local ingress = self, + + Default(name, ingressTls=[], ingressRules=[], labels=null):: + $.extensions.v1beta1.ApiVersion + + common.Kind("Ingress") + + common.Metadata($.v1.metadata.Default(name=name, labels=labels)) { + spec: { + tls: ingressTls, + rules: ingressRules, + }, + }, + + Metadata:: common.mixin.Metadata, + Spec(mixin):: {spec+: mixin}, + + spec:: { + Tls:: CreateTlsFunction(), + Rule:: CreateRuleFunction(), + }, + + rule:: { + Default:: CreateRuleFunction(), + }, + + httpIngressPath:: { + Default(serviceName, servicePort, path=null):: { + backend: { + serviceName: serviceName, + servicePort: servicePort, + }, + [if path != null then "path"]: path, + }, + }, + + mixin:: { + metadata:: common.mixin.metadata, + spec:: { + Tls:: CreateTlsFunction(function(tls) ingress.Spec({tls+: [tls]})), + Rule:: CreateRuleFunction( + function(rule) ingress.Spec({rules+: [rule]})), + }, + }, + + local CreateTlsFunction(createMixin=null) = + local tls(hosts, secretName) = { + [if hosts != null then "hosts"]: hosts, + [if secretName != null then "secretName"]: secretName, + }; + local partial = meta.MixinPartial2(tls, createMixin); + function(hosts=null, secretName=null) partial(hosts, secretName), + + local CreateRuleFunction(createMixin=null) = + local rule(host, httpIngressRule) = { + [if host != null then "host"]: host, + [if httpIngressRule != null then "http"]: httpIngressRule, + }; + local partial = meta.MixinPartial2(rule, createMixin); + function(host=null, http=null) partial(host, http), + }, + }, + }, + + meta:: { + v1:: { + labelSelector:: { + DefaultMatchLabelReqs(labels):: {matchLabels: labels}, + DefaultMatchExpressions(expressions):: {matchExpressions: expressions}, + }, + }, + }, + + policy:: { + v1beta1:: { + ApiVersion:: { apiVersion: "policy/v1beta1" }, + + podDistruptionBudget:: { + + Default(name, labels=null):: + $.policy.v1beta1.ApiVersion + + common.Kind("PodDisruptionBudget") + + common.Metadata($.v1.metadata.Default(name=name, labels=labels)) { + spec: {}, + }, + + Metadata:: common.mixin.Metadata, + Spec(mixin):: {spec+: mixin}, + + spec:: { + Selector:: $.extensions.v1beta1.deployment.Selector, + MinAvailable(time):: {minAvailable: time}, + }, + + mixin:: { + spec:: { + Selector:: meta.MixinPartial1( + $.extensions.v1beta1.deployment.spec.Selector, + $.extensions.v1beta1.deployment.Spec), + MinAvailable:: meta.MixinPartial1( + $.policy.v1beta1.podDistruptionBudget.spec.MinAvailable, + $.extensions.v1beta1.deployment.Spec), + }, + }, + }, + }, + }, + + apps:: { + v1beta1:: { + ApiVersion:: { apiVersion: "apps/v1beta1" }, + + statefulSet:: { + local statefulSet = self, + + Default( + name, replicas, template, serviceName=name, volumeClaimTemplates=[], + selector=null + ):: + $.apps.v1beta1.ApiVersion + + common.Kind("StatefulSet") + + common.Metadata($.v1.metadata.Default(name=name)) { + spec: statefulSet.spec.Default( + serviceName, replicas, template, volumeClaimTemplates, selector), + }, + + Metadata:: common.mixin.Metadata, + Spec(mixin):: {spec+: mixin}, + + spec:: { + Default( + serviceName, replicas, template, volumeClaimTemplates=[], + selector=null + ):: { + serviceName: serviceName, + replicas: replicas, + template: template, + volumeClaimTemplates: volumeClaimTemplates, + [if selector != null then "selector"]: selector, + }, + + ServiceName(serviceName):: {serviceName: serviceName}, + Template(podTemplate):: {template+: podTemplate}, + VolumeClaimTemplates(vcTemplates):: + {volumeClaimTemplates+: vcTemplates}, + // Selector(selector):: {selector: selector}, + }, + + mixin:: { + spec:: { + ServiceName:: meta.MixinPartial1( + $.apps.v1beta1.statefulSet.spec.ServiceName, + $.apps.v1beta1.statefulSet.Spec), + Template:: meta.MixinPartial1( + $.apps.v1beta1.statefulSet.spec.Template, + $.apps.v1beta1.statefulSet.Spec), + VolumeClaimTemplates:: meta.MixinPartial1( + $.apps.v1beta1.statefulSet.spec.VolumeClaimTemplates, + $.apps.v1beta1.statefulSet.Spec), + }, + }, + }, + }, + }, +} diff --git a/kpm/jsonnet/lib/internal/assert.libsonnet b/kpm/jsonnet/lib/internal/assert.libsonnet new file mode 100644 index 0000000..f865498 --- /dev/null +++ b/kpm/jsonnet/lib/internal/assert.libsonnet @@ -0,0 +1,25 @@ +{ + Type(fieldName, value, targetType): { + local observedType = std.type(value), + assert observedType == targetType + : "Field '%s' must be type '%s'; value was type '%s', with value '%s'" + % [fieldName, targetType, observedType, value] + }, + + InSet(fieldName, value, set): { + assert std.length(set) > 0 && std.type(set[0]) == std.type(value) + : "Field '%s' with value '%s' is of type '%s', but set '%s' contains elements of type '%s'" + % [fieldName, value, std.type(value), set, std.type(set[0])], + assert std.length(std.setInter(set, [value])) == 1 + : "Field '%s' with value '%s' must be in set '%s'" + % [fieldName, value, set] + }, + + ValidPort(fieldName, port): { + assert port > 0 && port < 65536 + : "Port '%s' must be in range 0 < port < 65536, but had value '%d'" + % [fieldName, port] + // NOTE: For some reason, Jsonnet only executes this check first + // if we put it after the port range assert. + } + self.Type(fieldName, port, "number"), +} \ No newline at end of file diff --git a/kpm/jsonnet/lib/internal/base.libsonnet b/kpm/jsonnet/lib/internal/base.libsonnet new file mode 100644 index 0000000..80fee7e --- /dev/null +++ b/kpm/jsonnet/lib/internal/base.libsonnet @@ -0,0 +1,15 @@ +{ + local baseName = "B98F6CE0-DEE9-43AE-BC09-C7C8EDE55029", + New(name, id): { + [baseName]:: { + name: name, + id: id, + }, + }, + + Verify(targetBase):: { + // assert super[baseName].id == targetBase[baseName].id + // : "Can't '+' object of type '%s' with object of type '%s'" + // % [super[baseName].name, targetBase[baseName].name] + }, +} \ No newline at end of file diff --git a/kpm/jsonnet/lib/internal/meta.libsonnet b/kpm/jsonnet/lib/internal/meta.libsonnet new file mode 100644 index 0000000..dd98c44 --- /dev/null +++ b/kpm/jsonnet/lib/internal/meta.libsonnet @@ -0,0 +1,11 @@ +{ + MixinPartial1(fn, createMixin=null):: + if createMixin != null + then function(arg1) createMixin(fn(arg1)) + else fn, + + MixinPartial2(fn, createMixin=null):: + if createMixin != null + then function(arg1, arg2) createMixin(fn(arg1, arg2)) + else fn, +} \ No newline at end of file diff --git a/kpm/jsonnet/lib/kpm-utils.libjsonnet b/kpm/jsonnet/lib/kpm-utils.libsonnet similarity index 100% rename from kpm/jsonnet/lib/kpm-utils.libjsonnet rename to kpm/jsonnet/lib/kpm-utils.libsonnet diff --git a/kpm/jsonnet/lib/kpm.libjsonnet b/kpm/jsonnet/lib/kpm.libsonnet similarity index 96% rename from kpm/jsonnet/lib/kpm.libjsonnet rename to kpm/jsonnet/lib/kpm.libsonnet index 2c9996e..e4532fa 100644 --- a/kpm/jsonnet/lib/kpm.libjsonnet +++ b/kpm/jsonnet/lib/kpm.libsonnet @@ -1,4 +1,4 @@ -local kpmstd = import "kpm-utils.libjsonnet"; +local kpmstd = import "kpm-utils.libsonnet"; kpmstd { local kpm = self, @@ -92,7 +92,9 @@ kpmstd { ), template(resource):: ( - local r = if std.objectHas(resource, "expander") == false || + local r = if std.objectHas(resource, "value") == true then + resource + else if std.objectHas(resource, "expander") == false || resource.expander == 'none' || resource.expander == null then resource { value: kpm.loadObject(resource.template) } else if resource.expander == "fmt" then diff --git a/kpm/jsonnet/lib/kube.libsonnet b/kpm/jsonnet/lib/kube.libsonnet new file mode 160000 index 0000000..c7713e7 --- /dev/null +++ b/kpm/jsonnet/lib/kube.libsonnet @@ -0,0 +1 @@ +Subproject commit c7713e7ee873e900eb079465ec3ae741d4eda658 diff --git a/kpm/jsonnet/lib/util.libsonnet b/kpm/jsonnet/lib/util.libsonnet new file mode 100644 index 0000000..f284f61 --- /dev/null +++ b/kpm/jsonnet/lib/util.libsonnet @@ -0,0 +1,90 @@ +local kubeAssert = import "internal/assert.libsonnet"; +local core = import "./core.libsonnet"; + +{ + app:: { + v1:: { + container:: { + NewWithPorts(name, image, ports):: + core.v1.container.Default(name, image) + + core.v1.container.Ports(ports), + }, + + env:: { + array:: { + // TODO: In all of these, check that we're not duplicating + // the variables, as the order is independent in Jsonnet, + // and we will mess it up. + + FromConfigMap(configMap, envSpec):: + self.FromConfigMapName(configMap.metadata.name, envSpec), + + FromConfigMapName(configMapName, envSpec):: + [core.v1.env.ValueFrom(name, configMapName, envSpec[name]) + for name in std.objectFields(envSpec)], + + FromSecret(secret, envSpec):: + self.FromSecretName(secret.metadata.name, envSpec), + + FromSecretName(secretName, envSpec):: + [core.v1.env.ValueFromSecret(name, secretName, envSpec[name]) + for name in std.objectFields(envSpec)], + + FromObj(envVariables):: + [core.v1.env.Variable(name, envVariables[name]) + for name in std.objectFields(envVariables)], + }, + }, + + pod:: { + FromContainer(container, labels={app: container.name}, volumes=[]):: + core.v1.pod.Default([container], volumes) + + core.v1.pod.Metadata(core.v1.metadata.Labels(labels)), + + template:: { + FromContainer(container, labels={app: container.name}, volumes=[]):: + core.v1.pod.template.Default([container], volumes) + + core.v1.pod.template.Metadata(core.v1.metadata.Labels(labels)), + }, + }, + + port:: { + service:: { + array:: { + FromContainerPorts(createServicePort, containerPorts):: [ + core.v1.port.service.Named( + port.name, createServicePort(port), port.name) + for port in containerPorts], + } + }, + }, + }, + + v1beta1:: { + deployment:: { + FromPodTemplate(name, replicas, podTemplate, labels={}):: + core.extensions.v1beta1.deployment.Default( + name, + core.extensions.v1beta1.deployment.spec.ReplicatedPod( + replicas, podTemplate)) + + core.extensions.v1beta1.deployment.Metadata( + core.v1.metadata.Labels(labels)), + + FromContainer( + name, + replicas, + container, + labels={}, + podLabels={app: container.name}, + volumes=[] + ):: + self.FromPodTemplate( + name, + replicas, + $.app.v1.pod.template.FromContainer( + container, labels=podLabels, volumes=volumes), + labels=labels), + }, + }, + }, +} \ No newline at end of file diff --git a/kpm/jsonnet/manifest.jsonnet.j2 b/kpm/jsonnet/manifest.jsonnet.j2 index 3544050..97af3f0 100644 --- a/kpm/jsonnet/manifest.jsonnet.j2 +++ b/kpm/jsonnet/manifest.jsonnet.j2 @@ -1,4 +1,4 @@ -local kpm = import "kpm.libjsonnet"; +local kpm = import "kpm.libsonnet"; function( params={} diff --git a/kpm/render_jsonnet.py b/kpm/render_jsonnet.py index d6fc1fd..64ecc89 100644 --- a/kpm/render_jsonnet.py +++ b/kpm/render_jsonnet.py @@ -1,13 +1,15 @@ -import yaml import json -import _jsonnet -import re -import os import logging +import os import os.path +import re + +import _jsonnet import jinja2 -from kpm.utils import convert_utf8 +import yaml + import kpm.template_filters as filters +from kpm.utils import convert_utf8 logger = logging.getLogger(__name__) @@ -20,9 +22,9 @@ def yaml_to_jsonnet(manifestyaml, tla_codes=None): jinja_env.filters.update(filters.jinja_filters()) # 1. Resolve old manifest variables # Load 'old' manifest.yaml - v = {"manifest": convert_utf8(json.loads(json.dumps(yaml.load(manifestyaml))))} + tempvars = {"manifest": convert_utf8(json.loads(json.dumps(yaml.load(manifestyaml))))} # Get variable from the 'old' manfiest and update them - variables = v['manifest'].get("variables", {}) + variables = tempvars['manifest'].get("variables", {}) if tla_codes is not None and 'params' in tla_codes: tla = json.loads(tla_codes['params']).get("variables", {}) variables.update(tla) @@ -53,20 +55,22 @@ def try_path(self, path, rel): if not rel: raise RuntimeError('Got invalid filename (empty string).') - if rel in ["kpm.libjsonnet", "kpm-utils.libjsonnet"]: - with open(os.path.join(os.path.dirname(__file__), "jsonnet/lib/%s" % rel)) as f: - return rel, f.read() + # @TODO(ant31) Search path for both for all files + if rel == "kpm.libjsonnet": + rel = "kpm.libsonnet" if self.files is not None and rel in self.files: if self.files[rel] is None: with open(rel) as f: self.files[rel] = f.read() return rel, self.files[rel] - elif self.manifestdir: + elif self.manifestdir and os.path.isfile(os.path.join(self.manifestdir, rel)): filepath = os.path.join(self.manifestdir, rel) - if os.path.isfile(filepath): - with open(filepath) as f: - return rel, f.read() + with open(filepath) as f: + return rel, f.read() + elif os.path.isfile(os.path.join(os.path.dirname(__file__), "jsonnet/lib/%s" % rel)): + with open(os.path.join(os.path.dirname(__file__), "jsonnet/lib/%s" % rel)) as f: + return rel, f.read() if rel[0] == '/': full_path = rel From f359d7949639c354cbbdb99344d6517f058cca0c Mon Sep 17 00:00:00 2001 From: Antoine Legrand <2t.antoine@gmail.com> Date: Thu, 4 May 2017 21:07:37 +0200 Subject: [PATCH 2/5] Add jsonnet for non-manifest.jsonnet files --- kpm/commands/jsonnet.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/kpm/commands/jsonnet.py b/kpm/commands/jsonnet.py index 6d68a0b..d77fe36 100644 --- a/kpm/commands/jsonnet.py +++ b/kpm/commands/jsonnet.py @@ -1,3 +1,4 @@ +import os import json from kpm.render_jsonnet import RenderJsonnet from kpm.commands.command_base import CommandBase, LoadVariables @@ -28,11 +29,14 @@ def _add_arguments(cls, parser): def _call(self): r = RenderJsonnet(manifestpath=self.filepath) - namespace = self.namespace - self.variables['namespace'] = namespace - tla_codes = {"variables": self.variables} + if os.path.basename(self.filepath) == "manifest.jsonnet": + namespace = self.namespace + self.variables['namespace'] = namespace + tla_codes = {"params": json.dumps({"variables": self.variables})} + else: + tla_codes = self.variables p = open(self.filepath).read() - self.result = r.render_jsonnet(p, tla_codes={"params": json.dumps(tla_codes)}) + self.result = r.render_jsonnet(p, tla_codes=tla_codes) def _render_dict(self): return self.result From 62e13d17e2a1b4bc9a067392bc8915202fefd97c Mon Sep 17 00:00:00 2001 From: Antoine Legrand <2t.antoine@gmail.com> Date: Fri, 5 May 2017 18:19:34 +0200 Subject: [PATCH 3/5] Add more helpers --- kpm/jsonnet/lib/core.libsonnet | 4 ++-- kpm/jsonnet/lib/kpm-utils.libsonnet | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/kpm/jsonnet/lib/core.libsonnet b/kpm/jsonnet/lib/core.libsonnet index 5420042..4fe864f 100644 --- a/kpm/jsonnet/lib/core.libsonnet +++ b/kpm/jsonnet/lib/core.libsonnet @@ -297,7 +297,7 @@ local meta = import "internal/meta.libsonnet"; }, configMap:: { - Default(namespace, configMapName, data): + Default(namespace, configMapName, data):: bases.ConfigMap + $.v1.ApiVersion + common.Kind("ConfigMap") + @@ -415,7 +415,7 @@ local meta = import "internal/meta.libsonnet"; // Claim. // claim:: { - DefaultPersistent(claimName, accessModes, size, namespace=null): + DefaultPersistent(claimName, accessModes, size, namespace=null):: local defaultMetadata = common.Metadata( $.v1.metadata.Default(namespace=namespace, name=claimName)); bases.PersistentVolumeClaim + diff --git a/kpm/jsonnet/lib/kpm-utils.libsonnet b/kpm/jsonnet/lib/kpm-utils.libsonnet index ff5ef1e..0fa37f1 100644 --- a/kpm/jsonnet/lib/kpm-utils.libsonnet +++ b/kpm/jsonnet/lib/kpm-utils.libsonnet @@ -71,5 +71,16 @@ [x for x in array if x != null] ), + objectValues(obj):: ( + local fields = std.objectFields(obj); + [obj[key] for key in fields] + ), + + + objectMap(func, obj):: ( + local fields = std.objectFields(obj); + {[key]: func(obj[key]) for key in fields} + ), + local initSeed = kpmutils.randAlpha(256), } From 323bed2732a5eb8750aa4dc5f8b51983f506e4d1 Mon Sep 17 00:00:00 2001 From: Antoine Legrand <2t.antoine@gmail.com> Date: Fri, 5 May 2017 18:27:13 +0200 Subject: [PATCH 4/5] Add kubecfg-poc example --- examples/logstash/Chart.jsonnet | 7 +++++ examples/logstash/manifest.jsonnet | 25 +++++++++++++++ examples/logstash/manifest.yaml | 31 +++++++++++++++++++ .../config/001-input-kinesis-stream.cfg | 10 ++++++ .../config/010-filter-quay-plain.cfg | 31 +++++++++++++++++++ .../config/100-output-elasticsearch.cfg | 7 +++++ .../templates/config/patterns/kubernetes | 4 +++ .../templates/logstash-configmap.jsonnet | 19 ++++++++++++ examples/logstash/values.jsonnet | 13 ++++++++ 9 files changed, 147 insertions(+) create mode 100644 examples/logstash/Chart.jsonnet create mode 100644 examples/logstash/manifest.jsonnet create mode 100644 examples/logstash/manifest.yaml create mode 100644 examples/logstash/templates/config/001-input-kinesis-stream.cfg create mode 100644 examples/logstash/templates/config/010-filter-quay-plain.cfg create mode 100644 examples/logstash/templates/config/100-output-elasticsearch.cfg create mode 100644 examples/logstash/templates/config/patterns/kubernetes create mode 100644 examples/logstash/templates/logstash-configmap.jsonnet create mode 100644 examples/logstash/values.jsonnet diff --git a/examples/logstash/Chart.jsonnet b/examples/logstash/Chart.jsonnet new file mode 100644 index 0000000..3807d48 --- /dev/null +++ b/examples/logstash/Chart.jsonnet @@ -0,0 +1,7 @@ +{ + name: "quay/elastic-logstash-app", + author: "Antoine Legrand", + version: "5.2.2-1", + description: "logstash", + license: "MIT", +} \ No newline at end of file diff --git a/examples/logstash/manifest.jsonnet b/examples/logstash/manifest.jsonnet new file mode 100644 index 0000000..0c9c3f8 --- /dev/null +++ b/examples/logstash/manifest.jsonnet @@ -0,0 +1,25 @@ +local kubeapi = import "kubeapi.libsonnet"; + +function( + params={}, +) + +kubeapi.render({ + local application = self, + // metadata import + package: import "Chart.jsonnet", + // default parametrized values + variables: (import "values.jsonnet")(params.variables), + // resources to deploy + resources: { + "logstash-svc.json": kubeapi.k8s.service.Create(name="logstash", ports=[5044]), + "logstash-configmap.json": (import "templates/logstash-configmap.jsonnet")(application.variables), + } +}, params) + + + + + + + diff --git a/examples/logstash/manifest.yaml b/examples/logstash/manifest.yaml new file mode 100644 index 0000000..000f6c7 --- /dev/null +++ b/examples/logstash/manifest.yaml @@ -0,0 +1,31 @@ +--- +package: + name: quay/elastic-logstash-app + author: Antoine Legrand + version: 5.2.2-5 + description: logstash + license: MIT + +variables: + image: quay.io/ant31/logstash:5.2.2 + elasticsearch_hosts: '"elasticsearch.{{namespace}}.svc.cluster.local:9200"' + logstash_conf_volume: + name: logstashconf + configMap: + name: logstash + +resources: + - file: logstash-configmap.yaml + name: logstash + type: configmap + + - file: logstash-deploy.yaml + name: logstash + type: rc + + - file: logstash-svc.yaml + name: logstash + type: svc + +deploy: + - name: $self diff --git a/examples/logstash/templates/config/001-input-kinesis-stream.cfg b/examples/logstash/templates/config/001-input-kinesis-stream.cfg new file mode 100644 index 0000000..8fb9414 --- /dev/null +++ b/examples/logstash/templates/config/001-input-kinesis-stream.cfg @@ -0,0 +1,10 @@ +input { + kinesis { + kinesis_stream_name => "{{stream_name}}" + # metrics => "cloudwatch" + checkpoint_interval_seconds => 10 + application_name => "{{stream_name}}" + # TODO(ant31): add type + # codec => json { } + } + } diff --git a/examples/logstash/templates/config/010-filter-quay-plain.cfg b/examples/logstash/templates/config/010-filter-quay-plain.cfg new file mode 100644 index 0000000..a82e1d6 --- /dev/null +++ b/examples/logstash/templates/config/010-filter-quay-plain.cfg @@ -0,0 +1,31 @@ + filter { + mutate { + rename => {"message" => "raw_log"} + } + json { + source => "raw_log" + } + grok { + match => { "raw_log" => "%{SYSLOGBASE2}%{GREEDYDATA:message}" } + } + + date { + match => [ "timestamp" , "ISO8601", "dd/MMM/YYYY:HH:mm:ss Z", "dd/MMM/YYYY/HH/mm/ss"] + } + if [program] == "nginx" { + grok { + match => {"message" => " %{IPORHOST:clientip} \(%{IPORHOST:clientip2}\) %{DATA} %{DATA} \[%{DATA}\] \"%{WORD:http_verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:http_code} (?:%{NUMBER:bytes}|-) (?:\"(?:%{URI:referrer}|-)\") %{QS:user_agent} \(%{BASE10NUM:duration}.*\)" + } + } + mutate { + convert => {"duration" => "float"} + } + geoip { + source => "clientip" + } + grok { + match => {"user_agent" => "docker/%{DATA:[docker][version]} go/%{DATA:[docker][goversion]} git-commit/%{DATA:[docker][commit]} kernel/%{DATA:[docker][kernel]} os/%{DATA:[docker][os]} arch/%{DATA:[docker][arch]}$" + } + } + } + } diff --git a/examples/logstash/templates/config/100-output-elasticsearch.cfg b/examples/logstash/templates/config/100-output-elasticsearch.cfg new file mode 100644 index 0000000..6c35bcf --- /dev/null +++ b/examples/logstash/templates/config/100-output-elasticsearch.cfg @@ -0,0 +1,7 @@ +output { + elasticsearch { + index => "logstash-k8s-%{+YYYY.MM.dd}" + hosts => [{{elasticsearch_hosts}}] + flush_size => 50 + } + } \ No newline at end of file diff --git a/examples/logstash/templates/config/patterns/kubernetes b/examples/logstash/templates/config/patterns/kubernetes new file mode 100644 index 0000000..44eafb8 --- /dev/null +++ b/examples/logstash/templates/config/patterns/kubernetes @@ -0,0 +1,4 @@ +KUBERNETES_POD_ID ([0-9a-z]{5}) +KUBERNETES_NAME (?:[0-9a-z\-\.]{1,255}?) +KUBERNETES_DEPLOY_ID [0-9]{10} +KUBERNETES_SOURCE %{KUBERNETES_NAME:[kubernetes][controller_name]}(-%{KUBERNETES_DEPLOY_ID:[kubernetes][deployment_id]})?(-%{KUBERNETES_POD_ID:[kubernetes][pod_id]})?_%{KUBERNETES_NAME:[kubernetes][namespace]}_%{GREEDYDATA:[kubernetes][container_name]}-%{DATA:[kubernetes][container_id]}.log \ No newline at end of file diff --git a/examples/logstash/templates/logstash-configmap.jsonnet b/examples/logstash/templates/logstash-configmap.jsonnet new file mode 100644 index 0000000..19c03e3 --- /dev/null +++ b/examples/logstash/templates/logstash-configmap.jsonnet @@ -0,0 +1,19 @@ +local kubeapi = import "kubeapi.libsonnet"; +local utils = kubeapi.utils; +local k8s = kubeapi.k8s; + +function (variables={}) +local expandConfig(conf) = utils.expanders.jinja2(conf, variables); + +local data = utils.objectMap(expandConfig, { + "001-input-kinesis-stream.cfg": + importstr "config/001-input-kinesis-stream.cfg", + "100-output-elasticsearch.cfg": + importstr "config/100-output-elasticsearch.cfg", + "010-filter-quay-plain.cfg": + importstr "config/010-filter-quay-plain.cfg", +}); + +k8s.configMap.New("logstash-indexer") + +k8s.configMap.mergeData(data) + diff --git a/examples/logstash/values.jsonnet b/examples/logstash/values.jsonnet new file mode 100644 index 0000000..0875788 --- /dev/null +++ b/examples/logstash/values.jsonnet @@ -0,0 +1,13 @@ +function(commandline_vars) + +{ + stream_name: "kinesis", + image: "quay.io/ant31/logstash:5.2.2", + elasticsearch_hosts: '"elasticsearch.%s.svc.cluster.local:9200"' % self.namespace, + logstash_conf_volume: { + name: "logstashconf", + configMap: { + name: "logstash", + }, + }, +} + commandline_vars \ No newline at end of file From eb2a9b2eee0c9eff43f422fabdf9df0d1758fc62 Mon Sep 17 00:00:00 2001 From: Antoine Legrand <2t.antoine@gmail.com> Date: Fri, 5 May 2017 19:32:54 +0200 Subject: [PATCH 5/5] Add kubeapi.libsonnet --- examples/logstash/manifest.jsonnet | 11 ++- kpm/jsonnet/lib/kubeapi.libsonnet | 120 +++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 kpm/jsonnet/lib/kubeapi.libsonnet diff --git a/examples/logstash/manifest.jsonnet b/examples/logstash/manifest.jsonnet index 0c9c3f8..f7b4315 100644 --- a/examples/logstash/manifest.jsonnet +++ b/examples/logstash/manifest.jsonnet @@ -1,4 +1,6 @@ -local kubeapi = import "kubeapi.libsonnet"; +local kubecfg = import "kubecfg.libsonnet"; +local opencompose = kubecfg.opencompose; + function( params={}, @@ -11,9 +13,10 @@ kubeapi.render({ // default parametrized values variables: (import "values.jsonnet")(params.variables), // resources to deploy - resources: { - "logstash-svc.json": kubeapi.k8s.service.Create(name="logstash", ports=[5044]), - "logstash-configmap.json": (import "templates/logstash-configmap.jsonnet")(application.variables), + resources: opencompose.createServices() + + } + {'deployment.json'+: addSideCar()) + } }, params) diff --git a/kpm/jsonnet/lib/kubeapi.libsonnet b/kpm/jsonnet/lib/kubeapi.libsonnet new file mode 100644 index 0000000..f23cc82 --- /dev/null +++ b/kpm/jsonnet/lib/kubeapi.libsonnet @@ -0,0 +1,120 @@ +local core = import "core.libsonnet"; +local kubeUtil = import "util.libsonnet"; +local kpm = import "kpm.libsonnet"; + +local utils = { + resource: { + getName(obj):: + obj['metadata']['name'], + getLabels(obj):: + obj['metadata']['labels'], + setNamespace(namespace):: + { metadata+: {namespace: namespace } }, + setName(name):: + { metadata+: {name: name} }, + }, + + expanders: { + jinja2:: kpm.jinja2, + jsonnet:: kpm.jsonnet, + }, + +} + kpm; + +local portUtils = { + portName(port):: "port-%s" % port, + containerPorts(ports):: + [core.v1.port.container.Named(self.portName(port), port) for port in ports], + servicePorts(ports):: + [core.v1.port.service.Named(self.portName(port), port, port) for port in ports], +}; + +local configMapUtil = { + Create(name, data={}):: + self.Name(name) + + self.mergeData(data), + New(name):: + core.v1.configMap.Default("default", name, data={}), + mergeData(data):: + {data+: data} +}; + +local serviceUtil = { + Create(name, ports=[], selector=null, type="ClusterIP"):: ( + local local_selector = + if selector == null then + {app: name} + else + selector; + self.New(name) + + self.setType(type) + + self.setSelector(local_selector) + + self.setPorts(ports)), + New(name):: + core.v1.service.Default(name, [],), + setType(type):: + core.v1.service.mixin.spec.Type(type), + setSelector(obj):: + core.v1.service.mixin.spec.Selector(obj), + _ports(ports):: [ + if std.type(port) == "number" then + core.v1.port.service.Named(portUtils.portName(port), port, port) + else + core.v1.port.service.Named( + if std.objectHas(port, 'name') then port.name else portUtils.portName(port.port), + port.port, + if std.objectHas(port, 'targetPort') then port.targetPort else port.port) + for port in ports + ], + setPorts(ports):: {spec+: {ports: serviceUtil._ports(ports)}}, + mergePorts(ports):: {spec+: {ports+: serviceUtil._ports(ports)}}, + +}; + +// This is CRAZY imports +local k8s_resources = { + deployment: core.extensions.v1beta1.deployment + kubeUtil.app.v1beta1.deployment, // Join util by default + container: core.v1.container, // get ride of the v1 + claim: core.v1.volume.claim, + probe: core.v1.probe, + pod: core.v1.pod + kubeUtil.app.v1.pod, + port: core.v1.port + kubeUtil.app.v1.port + portUtils, + service: core.v1.service + serviceUtil, + secret: core.v1.secret, + metadata: core.v1.metadata, + persistent: core.v1.volume.persistent, + volume: core.v1.volume, + configMap: core.v1.configMap + configMapUtil, + mount: core.v1.volume.mount, +}; +local render_utils = { + setResource(res, params):: ( + local resource = + if std.objectHas(res, 'content') == false then + {content: res} else res; + resource['content'] + + utils.resource.setNamespace(params.variables.namespace) + + if std.objectHas(resource, 'name') then + utils.resource.setName(resource['name']) else {} ), + + + createList(app, params):: { + apiVersion: "v1", + kind: "List", + items: [render_utils.setResource(resource, params) + for resource in utils.objectValues(app.resources)] + }, +}; + +local render(app, params) = ( + if std.objectHas(params.variables, 'action') && params.variables.action == "generate" then + render_utils.createList(app, params) + else + app +); + +{ + utils: utils, + k8s: k8s_resources, + render:: render, +}