diff --git a/firefly.yaml.example b/firefly.yaml.example index 3efd5f3..7451187 100644 --- a/firefly.yaml.example +++ b/firefly.yaml.example @@ -24,6 +24,7 @@ data_server: # The data sources to collect data from. # These are Python modules contained in data_sources/ data_sources: + - data_sources.librato.Librato - data_sources.ganglia_rrd.GangliaRRD - data_sources.stat_monster_rrd.StatMonsterRRD # - data_sources.graphite_wsp.GraphiteWSP @@ -48,6 +49,10 @@ data_server: data_sources.graphite_wsp.GraphiteWSP: # path to the root of Graphite whisper files graphite_storage: null + data_sources.librato.Librato: + librato_url: "https://metrics-api.librato.com/" + username: "librato_username" + password: "librato_password" # The location of the SQLite database file which contains the data store # for the DATA SERVER diff --git a/firefly/data_sources/librato.py b/firefly/data_sources/librato.py new file mode 100644 index 0000000..a94d69d --- /dev/null +++ b/firefly/data_sources/librato.py @@ -0,0 +1,121 @@ +from datetime import datetime +from datetime import timedelta +from itertools import izip +from urlparse import urljoin + +import requests + +try: + import simplejson as json +except ImportError: + import json + +import firefly.data_source + +class Librato(firefly.data_source.DataSource): + """ + Stats from the Librato API + """ + + DESC = "Librato" + + def __init__(self, *args, **kwargs): + super(Librato, self).__init__(*args, **kwargs) + self.librato_url = kwargs['librato_url'] + self.username = kwargs['username'] + self.password = kwargs['password'] + self.resolution = kwargs['resolution'] + self.offset = 100 + + def list_path(self, path): + query = '.'.join(path + ['*']) + metric_name = query.split('*')[0] + + contents = list() + + # Each metric in Librato has a number of sources, as opposed to Firefly where everything is a source + # In the case that there is a metric name in the path, get its sources + if metric_name: + params = { + 'start_time': (datetime.now() - timedelta(hours=1)).strftime('%s'), + 'resolution': self.resolution + } + + find_url = self._build_librato_metrics_url(metric_name[:-1], params) + + r = requests.get(find_url, auth=(self.username, self.password)) + find_results = r.json() + + for measurement in find_results['measurements']: + contents.extend(self._form_entries_from_file(measurement)) + + # Otherwise, get all the metrics + else: + offset = 0 + metrics_remaining = True + + # Librato can only send a 100 metrics at a time, so multiple requests are required to get all the metrics + while metrics_remaining: + + find_url = urljoin(self.librato_url, 'v1/metrics?name=' + query + '&offset=' + str(offset)) + + r = requests.get(find_url, auth=(self.username, self.password)) + find_results = r.json() + + offset += self.offset + + if find_results['metrics']: + for metric in find_results['metrics']: + contents.extend(self._form_entries_from_dir(metric['name'])) + else: + metrics_remaining = False + + return contents + + def _form_entries_from_dir(self, name): + return [{'type': 'dir', 'name': name, 'children': None}] + + def _form_entries_from_file(self, name): + return [{'type': 'file', 'name': name}] + + def data(self, sources, start, end, width): + + params = { + 'start_time': start, + 'end_time': end, + 'resolution': self.resolution + } + + metric_name = sources[0][0] + source = sources[0][1] + + render_url = self._build_librato_metrics_url(metric_name, params) + + r = requests.get(render_url, auth=(self.username, self.password)) + render_results = r.json() + + out = [] + + for result in render_results['measurements']: + # Librato shortens really long source names with a '..' + # So, we check if either the source name matches or the beginning and end of the shortened source name are substrings + if (source == result) or all(substr in result for substr in source.split('..')): + for datapoint in render_results['measurements'][source]: + point = {} + point['t'] = datapoint['measure_time'] + point['v'] = [datapoint['value']] + out.append(point) + break + + return json.dumps(out) + + def _build_librato_metrics_url(self, metric_name, params): + url_params = 'v1/metrics/' + metric_name + '?' + url_params += '&'.join(['%s=%s' % (k, v) for k, v in params.items()]) + return urljoin(self.librato_url, url_params) + + def legend(self, sources): + return self._svc(sources) + + def title(self, sources): + return ["librato"] diff --git a/requirements-dev.txt b/requirements-dev.txt index d8b3695..f78a2c5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,4 @@ PyHamcrest==1.8.0 testify==0.5.2 flake8 +requests>=2.3, <3.0 diff --git a/requirements.txt b/requirements.txt index cdeaa76..320fdd3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ -e . PyYAML==3.10 tornado>= 1.1, <2.0 +requests>= 2.3, < 3.0 diff --git a/setup.py b/setup.py index 101922a..d21b9f4 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ "pycurl", "pyyaml == 3.10", "tornado >= 1.1, <2.0", + "requests >=2.3, <3.0", ]