Added new parameter: convert_timestamp

bin/elasticsplunk.py

@@ -9,6 +9,9 @@
 import sys
 import json
 import time
+import calendar
+import dateparser
+from datetime import datetime
 from pprint import pprint
 from elasticsearch import Elasticsearch, helpers
 from splunklib.searchcommands import \
@@ -49,6 +52,7 @@
 KEY_CONFIG_INCLUDE_RAW = "include_raw"
 KEY_CONFIG_LIMIT = "limit"
 KEY_CONFIG_QUERY = "query"
+KEY_CONFIG_CONVERT_TIMESTAMP = "convert_timestamp"
 
 # Splunk keys
 KEY_SPLUNK_TIMESTAMP = "_time"
@@ -82,6 +86,7 @@ class ElasticSplunk(GeneratingCommand):
                       doc="Earliest event, format relative eg. now-4h or 2016-11-18T23:45:00")
     latest = Option(require=False, default=None,
                     doc="Latest event, format 2016-11-17T23:45:00")
+    convert_timestamp = Option(require=False, default=False, doc="Convert timestamps from text to unix timestamp")
 
     @staticmethod
     def parse_dates(time_value):
@@ -111,6 +116,13 @@ def parse_dates(time_value):
             return int(time.mktime(time.strptime(time_value, "%Y-%m-%dT%H:%M:%S")))
 
 
+    @staticmethod
+    def to_epoch(timestring):
+        """Convert UTC date string returned by elasticsearch to epoch"""
+        dt = dateparser.parse(timestring)
+       	utc_dt = dt.replace(tzInfo=None) - dt.utcoffset()
+	return (utc_dt - datetime(1970, 1, 1)).total_seconds()
+
     def _get_search_config(self):
         """Parse and configure search parameters"""
 
@@ -177,6 +189,7 @@ def _get_search_config(self):
         config[KEY_CONFIG_INCLUDE_RAW] = self.include_raw
         config[KEY_CONFIG_LIMIT] = self.limit
         config[KEY_CONFIG_QUERY] = self.query
+        config[KEY_CONFIG_CONVERT_TIMESTAMP] = True if self.convert_timestamp in [True, "true", "True", 1, "y"] else False
 
         return config
 
@@ -185,7 +198,10 @@ def _parse_hit(self, config, hit):
         """Parse a Elasticsearch Hit"""
 
         event = {}
-        event[KEY_SPLUNK_TIMESTAMP] = hit[KEY_ELASTIC_SOURCE][config[KEY_CONFIG_TIMESTAMP]]
+        if config[KEY_CONFIG_CONVERT_TIMESTAMP]:
+            event[KEY_SPLUNK_TIMESTAMP] = self.to_epoch(hit[KEY_ELASTIC_SOURCE][config[KEY_CONFIG_TIMESTAMP]])
+        else:
+            event[KEY_SPLUNK_TIMESTAMP] = hit[KEY_ELASTIC_SOURCE][config[KEY_CONFIG_TIMESTAMP]]
         for key in hit[KEY_ELASTIC_SOURCE]:
             if key != config[KEY_CONFIG_TIMESTAMP]:
                 if isinstance(hit[KEY_ELASTIC_SOURCE][key], dict):

default/searchbnf.conf

@@ -13,6 +13,6 @@ tags = search elasticsearch
 related = search
 
 [ess-options]
-syntax = eaddr=<string> | action=<string> | scan=<bool> | index=<string> | stype=<string> | tsfield=<string> | query=<string> | fields=<string> | limit=<int> | include_es=<bool> | include_raw=<bool>| earliest=<string> | earliest=<string> | latest=<latest>
+syntax = eaddr=<string> | action=<string> | scan=<bool> | index=<string> | stype=<string> | tsfield=<string> | query=<string> | fields=<string> | limit=<int> | include_es=<bool> | include_raw=<bool>| earliest=<string> | earliest=<string> | latest=<latest> | convert_timestamp=<bool>
 description = Search ElasticSearch within Splunk