From 7daaa3599bebb1a91376ec0ca2b80693e7d18d9b Mon Sep 17 00:00:00 2001 From: Akhil Mallavarapu Date: Thu, 14 Aug 2014 14:20:21 -0700 Subject: [PATCH 1/6] Created new data source for Librato and updated example YAML config --- firefly.yaml.example | 4 ++ firefly/data_sources/librato.py | 123 ++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 firefly/data_sources/librato.py diff --git a/firefly.yaml.example b/firefly.yaml.example index 3efd5f3..d11e6af 100644 --- a/firefly.yaml.example +++ b/firefly.yaml.example @@ -48,6 +48,10 @@ data_server: data_sources.graphite_wsp.GraphiteWSP: # path to the root of Graphite whisper files graphite_storage: null + data_sources.librato.Librato: + librato_url: "https://metrics-api.librato.com/" + username: "librato_username" + password: "librato_password" # The location of the SQLite database file which contains the data store # for the DATA SERVER diff --git a/firefly/data_sources/librato.py b/firefly/data_sources/librato.py new file mode 100644 index 0000000..bfbe137 --- /dev/null +++ b/firefly/data_sources/librato.py @@ -0,0 +1,123 @@ +from datetime import datetime, timedelta +from urllib2 import urlopen, Request +from itertools import izip +from urlparse import urljoin + +import base64 +import requests + +try: + import json +except ImportError: + import simplejson as json + +import firefly.data_source + +class Librato(firefly.data_source.DataSource): + """ + Stats from the Librato API + """ + + DESC = "Librato" + + def __init__(self, *args, **kwargs): + super(Librato, self).__init__(*args, **kwargs) + self.librato_url = kwargs['librato_url'] + self.username = kwargs['username'] + self.password = kwargs['password'] + + def list_path(self, path): + query = '.'.join(path + ['*']) + metric_name = query.split('*')[0] + + contents = list() + + # Each metric in Librato has a number of sources, as opposed to Firefly where everything is a source + # In the case that there is a metric name in the path, get its sources + if metric_name: + params = { + 'start_time': (datetime.now() - timedelta(hours=1)).strftime('%s'), + 'resolution': 60 + } + + fmt = '%Y-%m-%d %H:%M:%S' + from_str = datetime.fromtimestamp(float(params['start_time'])).strftime(fmt) + + find_url = urljoin(self.librato_url, 'v1/metrics/' + metric_name[:-1] + '?' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()])) + r = requests.get(find_url, auth=(self.username, self.password)) + find_results = r.json() + + for measurement in find_results['measurements']: + contents.extend(self._form_entries_from_file(measurement)) + + # Otherwise, get all the metrics + else: + offset = 0 + metrics_remaining = True + + # Librato can only send a 100 metrics at a time, so multiple requests are required to get all the metrics + while metrics_remaining: + + find_url = urljoin(self.librato_url, 'v1/metrics?name=' + query + '&offset=' + str(offset)) + + r = requests.get(find_url, auth=(self.username, self.password)) + find_results = r.json() + + offset += 100 + + if find_results['metrics']: + for metric in find_results['metrics']: + contents.extend(self._form_entries_from_dir(metric['name'])) + else: + metrics_remaining = False + + return contents + + def _form_entries_from_dir(self, name): + return [{'type': 'dir', 'name': name, 'children': None}] + + def _form_entries_from_file(self, name): + return [{'type': 'file', 'name': name}] + + def data(self, sources, start, end, width): + + serieses = [] + step = None + + fmt = '%Y-%m-%d %H:%M:%S' + from_str = datetime.fromtimestamp(start).strftime(fmt) + + params = { + 'start_time': start, + 'end_time': end, + 'resolution': 60 + } + + metric_name = sources[0][0] + source = sources[0][1] + + + render_url = urljoin(self.librato_url, 'v1/metrics/' + metric_name + '?' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()])) + + r = requests.get(render_url, auth=(self.username, self.password)) + render_results = r.json() + + out = [] + + for result in render_results['measurements']: + # Librato shorterns really long source names with a '..' + # So, we check if either the source name matches or the shortened source name is a substring + source_substrings = source.split('..') + if (source == result) or (source_substrings[0] in result and source_substrings[1] in result): + for datapoint in render_results['measurements'][source]: + out.append({'t': datapoint['measure_time'], 'v': [datapoint['value']]}) + break + + return json.dumps(out) + + def legend(self, sources): + return self._svc(sources) + + def title(self, sources): + return ["librato"] + From 646bfe88fe09cec33b86f98ab18b6b48cf69b6af Mon Sep 17 00:00:00 2001 From: Akhil Mallavarapu Date: Thu, 14 Aug 2014 14:23:16 -0700 Subject: [PATCH 2/6] Added Librato to data sources in the YAML --- firefly.yaml.example | 1 + 1 file changed, 1 insertion(+) diff --git a/firefly.yaml.example b/firefly.yaml.example index d11e6af..7451187 100644 --- a/firefly.yaml.example +++ b/firefly.yaml.example @@ -24,6 +24,7 @@ data_server: # The data sources to collect data from. # These are Python modules contained in data_sources/ data_sources: + - data_sources.librato.Librato - data_sources.ganglia_rrd.GangliaRRD - data_sources.stat_monster_rrd.StatMonsterRRD # - data_sources.graphite_wsp.GraphiteWSP From 91ab5e18121686c642bead1e6f8c85fedab138b7 Mon Sep 17 00:00:00 2001 From: Akhil Mallavarapu Date: Thu, 14 Aug 2014 15:48:35 -0700 Subject: [PATCH 3/6] Fixed things from code review --- firefly/data_sources/librato.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/firefly/data_sources/librato.py b/firefly/data_sources/librato.py index bfbe137..4cc5b2a 100644 --- a/firefly/data_sources/librato.py +++ b/firefly/data_sources/librato.py @@ -1,9 +1,8 @@ -from datetime import datetime, timedelta -from urllib2 import urlopen, Request +from datetime import datetime +from datetime import timedelta from itertools import izip from urlparse import urljoin -import base64 import requests try: @@ -25,6 +24,7 @@ def __init__(self, *args, **kwargs): self.librato_url = kwargs['librato_url'] self.username = kwargs['username'] self.password = kwargs['password'] + self.resolution = 60 def list_path(self, path): query = '.'.join(path + ['*']) @@ -37,13 +37,11 @@ def list_path(self, path): if metric_name: params = { 'start_time': (datetime.now() - timedelta(hours=1)).strftime('%s'), - 'resolution': 60 + 'resolution': self.resolution } - fmt = '%Y-%m-%d %H:%M:%S' - from_str = datetime.fromtimestamp(float(params['start_time'])).strftime(fmt) - - find_url = urljoin(self.librato_url, 'v1/metrics/' + metric_name[:-1] + '?' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()])) + params = 'v1/metrics/' + metric_name[:-1] + '?' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()]) + find_url = urljoin(self.librato_url, params) r = requests.get(find_url, auth=(self.username, self.password)) find_results = r.json() @@ -81,23 +79,18 @@ def _form_entries_from_file(self, name): def data(self, sources, start, end, width): - serieses = [] - step = None - - fmt = '%Y-%m-%d %H:%M:%S' - from_str = datetime.fromtimestamp(start).strftime(fmt) - params = { 'start_time': start, 'end_time': end, - 'resolution': 60 + 'resolution': self.resolution } metric_name = sources[0][0] source = sources[0][1] - render_url = urljoin(self.librato_url, 'v1/metrics/' + metric_name + '?' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()])) + params = 'v1/metrics/' + metric_name + '?' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()]) + render_url = urljoin(self.librato_url, params) r = requests.get(render_url, auth=(self.username, self.password)) render_results = r.json() From 3b838ca9133893fcb57a7c09d35c9c56a2944d5b Mon Sep 17 00:00:00 2001 From: Akhil Mallavarapu Date: Mon, 18 Aug 2014 12:11:13 -0700 Subject: [PATCH 4/6] Updated requirements.txt --- requirements-dev.txt | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index d8b3695..2080549 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,4 @@ PyHamcrest==1.8.0 testify==0.5.2 flake8 +requests diff --git a/requirements.txt b/requirements.txt index cdeaa76..54e3a22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ -e . PyYAML==3.10 tornado>= 1.1, <2.0 +requests From b0f87c67926f22459410b147a4c1b4b2fcaf6144 Mon Sep 17 00:00:00 2001 From: Akhil Mallavarapu Date: Tue, 19 Aug 2014 15:20:27 -0700 Subject: [PATCH 5/6] Code review changes --- firefly/data_sources/librato.py | 39 +++++++++++++++++++-------------- requirements-dev.txt | 2 +- requirements.txt | 2 +- setup.py | 1 + 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/firefly/data_sources/librato.py b/firefly/data_sources/librato.py index 4cc5b2a..f558e93 100644 --- a/firefly/data_sources/librato.py +++ b/firefly/data_sources/librato.py @@ -6,9 +6,9 @@ import requests try: - import json -except ImportError: import simplejson as json +except ImportError: + import json import firefly.data_source @@ -24,8 +24,9 @@ def __init__(self, *args, **kwargs): self.librato_url = kwargs['librato_url'] self.username = kwargs['username'] self.password = kwargs['password'] - self.resolution = 60 - + self.resolution = kwargs['resolution'] + self.offset = 100 + def list_path(self, path): query = '.'.join(path + ['*']) metric_name = query.split('*')[0] @@ -39,9 +40,9 @@ def list_path(self, path): 'start_time': (datetime.now() - timedelta(hours=1)).strftime('%s'), 'resolution': self.resolution } - - params = 'v1/metrics/' + metric_name[:-1] + '?' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()]) - find_url = urljoin(self.librato_url, params) + + find_url = self._build_librato_metrics_url(metric_name[:-1], params) + r = requests.get(find_url, auth=(self.username, self.password)) find_results = r.json() @@ -61,7 +62,7 @@ def list_path(self, path): r = requests.get(find_url, auth=(self.username, self.password)) find_results = r.json() - offset += 100 + offset += self.offset if find_results['metrics']: for metric in find_results['metrics']: @@ -87,10 +88,8 @@ def data(self, sources, start, end, width): metric_name = sources[0][0] source = sources[0][1] - - - params = 'v1/metrics/' + metric_name + '?' + '&'.join(['%s=%s' % (k, v) for k, v in params.items()]) - render_url = urljoin(self.librato_url, params) + + render_url = self._build_librato_metrics_url(metric_name, params) r = requests.get(render_url, auth=(self.username, self.password)) render_results = r.json() @@ -99,18 +98,24 @@ def data(self, sources, start, end, width): for result in render_results['measurements']: # Librato shorterns really long source names with a '..' - # So, we check if either the source name matches or the shortened source name is a substring - source_substrings = source.split('..') - if (source == result) or (source_substrings[0] in result and source_substrings[1] in result): + # So, we check if either the source name matches or the beginning and end of the shortened source name are substrings + if (source == result) or all(substr in result for substr in source.split('..')): for datapoint in render_results['measurements'][source]: - out.append({'t': datapoint['measure_time'], 'v': [datapoint['value']]}) + point = {} + point['t'] = datapoint['measure_time'] + point['v'] = [datapoint['value']] + out.append(point) break return json.dumps(out) + + def _build_librato_metrics_url(self, metric_name, params): + url_params = 'v1/metrics/' + metric_name + '?' + url_params += '&'.join(['%s=%s' % (k, v) for k, v in params.items()]) + return urljoin(self.librato_url, url_params) def legend(self, sources): return self._svc(sources) def title(self, sources): return ["librato"] - diff --git a/requirements-dev.txt b/requirements-dev.txt index 2080549..f78a2c5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,4 +2,4 @@ PyHamcrest==1.8.0 testify==0.5.2 flake8 -requests +requests>=2.3, <3.0 diff --git a/requirements.txt b/requirements.txt index 54e3a22..320fdd3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ -e . PyYAML==3.10 tornado>= 1.1, <2.0 -requests +requests>= 2.3, < 3.0 diff --git a/setup.py b/setup.py index 101922a..d21b9f4 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ "pycurl", "pyyaml == 3.10", "tornado >= 1.1, <2.0", + "requests >=2.3, <3.0", ] From b7a553e2d342962d2322c998e69ebd1df04b4863 Mon Sep 17 00:00:00 2001 From: Akhil Mallavarapu Date: Wed, 20 Aug 2014 11:04:35 -0700 Subject: [PATCH 6/6] fixed typo in comment --- firefly/data_sources/librato.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firefly/data_sources/librato.py b/firefly/data_sources/librato.py index f558e93..a94d69d 100644 --- a/firefly/data_sources/librato.py +++ b/firefly/data_sources/librato.py @@ -97,7 +97,7 @@ def data(self, sources, start, end, width): out = [] for result in render_results['measurements']: - # Librato shorterns really long source names with a '..' + # Librato shortens really long source names with a '..' # So, we check if either the source name matches or the beginning and end of the shortened source name are substrings if (source == result) or all(substr in result for substr in source.split('..')): for datapoint in render_results['measurements'][source]: