Kubesonnet

.style.yapf

@@ -11,8 +11,7 @@ ALLOW_MULTILINE_DICTIONARY_KEYS=True
 # False
 BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=True
 BLANK_LINE_BEFORE_CLASS_DOCSTRING=False
-# False
-COALESCE_BRACKETS=False
+COALESCE_BRACKETS=True
 
 CONTINUATION_INDENT_WIDTH=4
 DEDENT_CLOSING_BRACKETS=False

MANIFEST.in

@@ -4,10 +4,7 @@ include LICENSE
 include README.md
 include requirements.txt
 include requirements_dev.txt
-include kpm/jsonnet/*.jsonnet
-include kpm/jsonnet/*.libjsonnet
-include kpm/jsonnet/lib/*.libjsonnet
-include kpm/jsonnet/manifest.jsonnet.j2
+recursive-include kpm/jsonnet *
 recursive-include tests *
 recursive-exclude * __pycache__
 recursive-exclude * *.py[co]

examples/logstash/Chart.jsonnet

@@ -0,0 +1,7 @@
+{
+          name: "quay/elastic-logstash-app",
+          author: "Antoine Legrand",
+          version: "5.2.2-1",
+          description: "logstash",
+          license: "MIT",
+}

examples/logstash/manifest.jsonnet

@@ -0,0 +1,28 @@
+local kubecfg = import "kubecfg.libsonnet";
+local opencompose = kubecfg.opencompose;
+
+
+function(
+  params={},
+)
+
+kubeapi.render({
+    local application = self,
+    // metadata import
+    package: import "Chart.jsonnet",
+    // default parametrized values
+    variables: (import "values.jsonnet")(params.variables),
+    // resources to deploy
+    resources: opencompose.createServices()
+
+    } + {'deployment.json'+: addSideCar())
+
+    }
+}, params)
+
+
+
+
+
+
+

examples/logstash/manifest.yaml

@@ -0,0 +1,31 @@
+---
+package:
+  name: quay/elastic-logstash-app
+  author: Antoine Legrand
+  version: 5.2.2-5
+  description: logstash
+  license: MIT
+
+variables:
+  image: quay.io/ant31/logstash:5.2.2
+  elasticsearch_hosts: '"elasticsearch.{{namespace}}.svc.cluster.local:9200"'
+  logstash_conf_volume:
+    name: logstashconf
+    configMap:
+      name: logstash
+
+resources:
+  - file: logstash-configmap.yaml
+    name: logstash
+    type: configmap
+
+  - file: logstash-deploy.yaml
+    name: logstash
+    type: rc
+
+  - file: logstash-svc.yaml
+    name: logstash
+    type: svc
+
+deploy:
+  - name: $self

examples/logstash/templates/config/001-input-kinesis-stream.cfg

@@ -0,0 +1,10 @@
+input {
+       kinesis {
+         kinesis_stream_name => "{{stream_name}}"
+         # metrics => "cloudwatch"
+         checkpoint_interval_seconds => 10
+         application_name => "{{stream_name}}"
+         # TODO(ant31): add type
+         # codec => json { }
+       }
+      }

examples/logstash/templates/config/010-filter-quay-plain.cfg

@@ -0,0 +1,31 @@
+  filter {
+          mutate {
+              rename => {"message" => "raw_log"}
+          }
+          json {
+            source => "raw_log"
+          }
+          grok {
+              match => { "raw_log" => "%{SYSLOGBASE2}%{GREEDYDATA:message}" }
+          }
+
+          date {
+              match => [ "timestamp" , "ISO8601", "dd/MMM/YYYY:HH:mm:ss Z", "dd/MMM/YYYY/HH/mm/ss"]
+          }
+          if [program] == "nginx" {
+              grok {
+                  match => {"message" => " %{IPORHOST:clientip} \(%{IPORHOST:clientip2}\) %{DATA} %{DATA} \[%{DATA}\] \"%{WORD:http_verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:http_code} (?:%{NUMBER:bytes}|-) (?:\"(?:%{URI:referrer}|-)\") %{QS:user_agent} \(%{BASE10NUM:duration}.*\)"
+                  }
+              }
+              mutate {
+                  convert => {"duration" => "float"}
+              }
+              geoip {
+                  source => "clientip"
+              }
+              grok {
+                  match => {"user_agent" => "docker/%{DATA:[docker][version]} go/%{DATA:[docker][goversion]} git-commit/%{DATA:[docker][commit]} kernel/%{DATA:[docker][kernel]} os/%{DATA:[docker][os]} arch/%{DATA:[docker][arch]}$"
+                  }
+              }
+          }
+      }

examples/logstash/templates/config/100-output-elasticsearch.cfg

@@ -0,0 +1,7 @@
+output {
+      elasticsearch {
+        index => "logstash-k8s-%{+YYYY.MM.dd}"
+        hosts => [{{elasticsearch_hosts}}]
+        flush_size => 50
+        }
+     }

examples/logstash/templates/config/patterns/kubernetes

@@ -0,0 +1,4 @@
+KUBERNETES_POD_ID ([0-9a-z]{5})
+KUBERNETES_NAME (?:[0-9a-z\-\.]{1,255}?)
+KUBERNETES_DEPLOY_ID [0-9]{10}
+KUBERNETES_SOURCE %{KUBERNETES_NAME:[kubernetes][controller_name]}(-%{KUBERNETES_DEPLOY_ID:[kubernetes][deployment_id]})?(-%{KUBERNETES_POD_ID:[kubernetes][pod_id]})?_%{KUBERNETES_NAME:[kubernetes][namespace]}_%{GREEDYDATA:[kubernetes][container_name]}-%{DATA:[kubernetes][container_id]}.log

examples/logstash/templates/logstash-configmap.jsonnet

@@ -0,0 +1,19 @@
+local kubeapi = import "kubeapi.libsonnet";
+local utils = kubeapi.utils;
+local k8s = kubeapi.k8s;
+
+function (variables={})
+local expandConfig(conf) = utils.expanders.jinja2(conf, variables);
+
+local data = utils.objectMap(expandConfig, {
+        "001-input-kinesis-stream.cfg":
+                importstr "config/001-input-kinesis-stream.cfg",
+        "100-output-elasticsearch.cfg":
+                importstr "config/100-output-elasticsearch.cfg",
+        "010-filter-quay-plain.cfg":
+                importstr "config/010-filter-quay-plain.cfg",
+});
+
+k8s.configMap.New("logstash-indexer") +
+k8s.configMap.mergeData(data)
+

examples/logstash/values.jsonnet

@@ -0,0 +1,13 @@
+function(commandline_vars)
+
+{
+  stream_name: "kinesis",
+  image: "quay.io/ant31/logstash:5.2.2",
+  elasticsearch_hosts: '"elasticsearch.%s.svc.cluster.local:9200"' % self.namespace,
+  logstash_conf_volume: {
+    name: "logstashconf",
+    configMap: {
+      name: "logstash",
+    },
+  },
+} + commandline_vars

kpm/commands/jsonnet.py

@@ -1,3 +1,4 @@
+import os
 import json
 from kpm.render_jsonnet import RenderJsonnet
 from kpm.commands.command_base import CommandBase, LoadVariables
@@ -28,11 +29,14 @@ def _add_arguments(cls, parser):
 
     def _call(self):
         r = RenderJsonnet(manifestpath=self.filepath)
-        namespace = self.namespace
-        self.variables['namespace'] = namespace
-        tla_codes = {"variables": self.variables}
+        if os.path.basename(self.filepath) == "manifest.jsonnet":
+            namespace = self.namespace
+            self.variables['namespace'] = namespace
+            tla_codes = {"params": json.dumps({"variables": self.variables})}
+        else:
+            tla_codes = self.variables
         p = open(self.filepath).read()
-        self.result = r.render_jsonnet(p, tla_codes={"params": json.dumps(tla_codes)})
+        self.result = r.render_jsonnet(p, tla_codes=tla_codes)
 
     def _render_dict(self):
         return self.result