diff --git a/bin/elasticsplunk.py b/bin/elasticsplunk.py index 1f12212..d9f39a2 100755 --- a/bin/elasticsplunk.py +++ b/bin/elasticsplunk.py @@ -9,6 +9,9 @@ import sys import json import time +import calendar +import dateparser +from datetime import datetime from pprint import pprint from elasticsearch import Elasticsearch, helpers from splunklib.searchcommands import \ @@ -49,6 +52,7 @@ KEY_CONFIG_INCLUDE_RAW = "include_raw" KEY_CONFIG_LIMIT = "limit" KEY_CONFIG_QUERY = "query" +KEY_CONFIG_CONVERT_TIMESTAMP = "convert_timestamp" # Splunk keys KEY_SPLUNK_TIMESTAMP = "_time" @@ -82,6 +86,7 @@ class ElasticSplunk(GeneratingCommand): doc="Earliest event, format relative eg. now-4h or 2016-11-18T23:45:00") latest = Option(require=False, default=None, doc="Latest event, format 2016-11-17T23:45:00") + convert_timestamp = Option(require=False, default=False, doc="Convert timestamps from text to unix timestamp") @staticmethod def parse_dates(time_value): @@ -111,6 +116,13 @@ def parse_dates(time_value): return int(time.mktime(time.strptime(time_value, "%Y-%m-%dT%H:%M:%S"))) + @staticmethod + def to_epoch(timestring): + """Convert UTC date string returned by elasticsearch to epoch""" + dt = dateparser.parse(timestring) + utc_dt = dt.replace(tzInfo=None) - dt.utcoffset() + return (utc_dt - datetime(1970, 1, 1)).total_seconds() + def _get_search_config(self): """Parse and configure search parameters""" @@ -177,6 +189,7 @@ def _get_search_config(self): config[KEY_CONFIG_INCLUDE_RAW] = self.include_raw config[KEY_CONFIG_LIMIT] = self.limit config[KEY_CONFIG_QUERY] = self.query + config[KEY_CONFIG_CONVERT_TIMESTAMP] = True if self.convert_timestamp in [True, "true", "True", 1, "y"] else False return config @@ -185,7 +198,10 @@ def _parse_hit(self, config, hit): """Parse a Elasticsearch Hit""" event = {} - event[KEY_SPLUNK_TIMESTAMP] = hit[KEY_ELASTIC_SOURCE][config[KEY_CONFIG_TIMESTAMP]] + if config[KEY_CONFIG_CONVERT_TIMESTAMP]: + event[KEY_SPLUNK_TIMESTAMP] = self.to_epoch(hit[KEY_ELASTIC_SOURCE][config[KEY_CONFIG_TIMESTAMP]]) + else: + event[KEY_SPLUNK_TIMESTAMP] = hit[KEY_ELASTIC_SOURCE][config[KEY_CONFIG_TIMESTAMP]] for key in hit[KEY_ELASTIC_SOURCE]: if key != config[KEY_CONFIG_TIMESTAMP]: if isinstance(hit[KEY_ELASTIC_SOURCE][key], dict): diff --git a/default/searchbnf.conf b/default/searchbnf.conf index 23678ca..8fbdb59 100644 --- a/default/searchbnf.conf +++ b/default/searchbnf.conf @@ -13,6 +13,6 @@ tags = search elasticsearch related = search [ess-options] -syntax = eaddr= | action= | scan= | index= | stype= | tsfield= | query= | fields= | limit= | include_es= | include_raw=| earliest= | earliest= | latest= +syntax = eaddr= | action= | scan= | index= | stype= | tsfield= | query= | fields= | limit= | include_es= | include_raw=| earliest= | earliest= | latest= | convert_timestamp= description = Search ElasticSearch within Splunk