From 76cd98441eacbac8144c5753dcc0714a447c7e28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vegard=20W=C3=A6rp?= <vegardw@gmail.com>
Date: Mon, 10 Sep 2018 20:29:45 +0200
Subject: [PATCH 1/2] Added new parameter: convert_timestamp

When elasticsearch returns timestamp fields, like
@timestamp from filebeats/logstash etc, they are returned as
strings formatted like '2018-08-28T06:55:23.471Z' (UTC time)
These strings are displayed OK by elasticsearch, but can't be
used by splunk commands like timechart, which need the timestamp to
be in unix epoch time format. This command converts the timestamp
to this format.

Uses calendar.timegm(), since that expects the time as UTC, while
time.mktime() expects local time. Concatenate with microseconds part
when returning to not loose precision as calendar.timegm() returns a integer
---
 bin/elasticsplunk.py   | 17 ++++++++++++++++-
 default/searchbnf.conf |  2 +-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/bin/elasticsplunk.py b/bin/elasticsplunk.py
index 1f12212..960a1be 100755
--- a/bin/elasticsplunk.py
+++ b/bin/elasticsplunk.py
@@ -9,6 +9,8 @@
 import sys
 import json
 import time
+import calendar
+from datetime import datetime
 from pprint import pprint
 from elasticsearch import Elasticsearch, helpers
 from splunklib.searchcommands import \
@@ -49,6 +51,7 @@
 KEY_CONFIG_INCLUDE_RAW = "include_raw"
 KEY_CONFIG_LIMIT = "limit"
 KEY_CONFIG_QUERY = "query"
+KEY_CONFIG_CONVERT_TIMESTAMP = "convert_timestamp"
 
 # Splunk keys
 KEY_SPLUNK_TIMESTAMP = "_time"
@@ -82,6 +85,7 @@ class ElasticSplunk(GeneratingCommand):
                       doc="Earliest event, format relative eg. now-4h or 2016-11-18T23:45:00")
     latest = Option(require=False, default=None,
                     doc="Latest event, format 2016-11-17T23:45:00")
+    convert_timestamp = Option(require=False, default=False, doc="Convert timestamps from text to unix timestamp")
 
     @staticmethod
     def parse_dates(time_value):
@@ -111,6 +115,13 @@ def parse_dates(time_value):
             return int(time.mktime(time.strptime(time_value, "%Y-%m-%dT%H:%M:%S")))
 
 
+    @staticmethod
+    def to_epoch(timestring):
+        """Convert UTC date string returned by elasticsearch to epoch"""
+        dt = datetime.strptime(timestring, "%Y-%m-%dT%H:%M:%S.%fZ")
+        return str(calendar.timegm(dt.timetuple())) + "." + str(dt.microsecond)
+
+
     def _get_search_config(self):
         """Parse and configure search parameters"""
 
@@ -177,6 +188,7 @@ def _get_search_config(self):
         config[KEY_CONFIG_INCLUDE_RAW] = self.include_raw
         config[KEY_CONFIG_LIMIT] = self.limit
         config[KEY_CONFIG_QUERY] = self.query
+        config[KEY_CONFIG_CONVERT_TIMESTAMP] = True if self.convert_timestamp in [True, "true", "True", 1, "y"] else False
 
         return config
 
@@ -185,7 +197,10 @@ def _parse_hit(self, config, hit):
         """Parse a Elasticsearch Hit"""
 
         event = {}
-        event[KEY_SPLUNK_TIMESTAMP] = hit[KEY_ELASTIC_SOURCE][config[KEY_CONFIG_TIMESTAMP]]
+        if config[KEY_CONFIG_CONVERT_TIMESTAMP]:
+            event[KEY_SPLUNK_TIMESTAMP] = self.to_epoch(hit[KEY_ELASTIC_SOURCE][config[KEY_CONFIG_TIMESTAMP]])
+        else:
+            event[KEY_SPLUNK_TIMESTAMP] = hit[KEY_ELASTIC_SOURCE][config[KEY_CONFIG_TIMESTAMP]]
         for key in hit[KEY_ELASTIC_SOURCE]:
             if key != config[KEY_CONFIG_TIMESTAMP]:
                 if isinstance(hit[KEY_ELASTIC_SOURCE][key], dict):
diff --git a/default/searchbnf.conf b/default/searchbnf.conf
index 23678ca..8fbdb59 100644
--- a/default/searchbnf.conf
+++ b/default/searchbnf.conf
@@ -13,6 +13,6 @@ tags = search elasticsearch
 related = search
 
 [ess-options]
-syntax = eaddr=<string> | action=<string> | scan=<bool> | index=<string> | stype=<string> | tsfield=<string> | query=<string> | fields=<string> | limit=<int> | include_es=<bool> | include_raw=<bool>| earliest=<string> | earliest=<string> | latest=<latest>
+syntax = eaddr=<string> | action=<string> | scan=<bool> | index=<string> | stype=<string> | tsfield=<string> | query=<string> | fields=<string> | limit=<int> | include_es=<bool> | include_raw=<bool>| earliest=<string> | earliest=<string> | latest=<latest> | convert_timestamp=<bool>
 description = Search ElasticSearch within Splunk
 

From 65c61a44fd86c42af821341d44978a7934189b6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vegard=20W=C3=A6rp?= <vegardw@gmail.com>
Date: Fri, 14 Sep 2018 15:06:49 +0200
Subject: [PATCH 2/2] Use dateparser module for parsing timestamp received from
 elasticsearch

---
 bin/elasticsplunk.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/bin/elasticsplunk.py b/bin/elasticsplunk.py
index 960a1be..d9f39a2 100755
--- a/bin/elasticsplunk.py
+++ b/bin/elasticsplunk.py
@@ -10,6 +10,7 @@
 import json
 import time
 import calendar
+import dateparser
 from datetime import datetime
 from pprint import pprint
 from elasticsearch import Elasticsearch, helpers
@@ -118,9 +119,9 @@ def parse_dates(time_value):
     @staticmethod
     def to_epoch(timestring):
         """Convert UTC date string returned by elasticsearch to epoch"""
-        dt = datetime.strptime(timestring, "%Y-%m-%dT%H:%M:%S.%fZ")
-        return str(calendar.timegm(dt.timetuple())) + "." + str(dt.microsecond)
-
+        dt = dateparser.parse(timestring)
+       	utc_dt = dt.replace(tzInfo=None) - dt.utcoffset()
+	return (utc_dt - datetime(1970, 1, 1)).total_seconds()
 
     def _get_search_config(self):
         """Parse and configure search parameters"""