Merge pull request #6 from statful/sampleRate

jmtavares · web-flow · commit aebf93c4e7af · 2016-11-22T12:21:07.000Z
feat: add sampleRate
diff --git a/README.md b/README.md
@@ -68,6 +68,7 @@ The custom options that can be set on config param are detailed below.
 | debug | Defines logs should be sent to console. | `boolean` | `false` | **NO** |
 | app | Defines the application global name. If specified sets a global tag `app=setValue`. | `string` | **undefined** | **NO** |
 | namespace | Defines the global namespace. | `string` | `web` | **NO** |
+| sampleRate | Defines the rate sampling. **Should be a number between [1, 100]**. | `number` | `100` | **NO** |
 | flushInterval | Defines the periodicity of buffer flushes in **miliseconds**. | `number` | `10000` | **NO** |
 | timeout | Defines the timeout. | `number` | `2000` | **NO** |
 | tags | Defines the global tags. | `object` | `{}` | **NO** |
diff --git a/bower.json b/bower.json
@@ -1,6 +1,6 @@
 {
   "name": "statful-client-javascript",
-  "version": "1.1.5",
+  "version": "1.1.6",
   "description": "Statful client for Javascript applications",
   "main": "dist/statful.js",
   "authors": [
diff --git a/dist/statful.js b/dist/statful.js
@@ -1,5 +1,5 @@
 /**
-* statful-client-javascript 1.1.5
+* statful-client-javascript 1.1.6
 * Copyright 2016 Statful <https://www.statful.com/>
 */
 (function(window) {
@@ -128,8 +128,13 @@
          * @param {object} item - object to be sent
          */
         this.addItemToQueue = function(queueName, item) {
-            if (this.listQueues[queueName]) {
+            var sampleRateNormalized = (item.sampleRate || this.config.sampleRate || 100) / 100;
+            if (this.listQueues[queueName] && Math.random() <= sampleRateNormalized) {
                 this.listQueues[queueName].data.push(item);
+                return true;
+            } else {
+                logger.debug("Metric was discarded due to sample rate.");
+                return false;
             }
         };
         /**
@@ -262,7 +267,8 @@
             aggregations: [ "last" ]
         },
         timeout: 2e3,
-        flushInterval: 1e4
+        flushInterval: 1e4,
+        sampleRate: 100
     };
     var logger;
     var statful = {
@@ -291,15 +297,16 @@
                     self.config[key] = clientConfig[key];
                 });
             };
-            self.metricsData = function(name, type, value, tags, aggregations, aggregationFrequency, namespace) {
+            self.metricsData = function(name, type, value, tags, aggregations, aggregationFrequency, namespace, sampleRate) {
                 return {
                     name: name,
                     type: type,
                     value: value,
                     tags: self.util.setTags(tags || {}, self.config.tags, self.config[type].tags, self.config.app),
                     aggregations: self.util.setAggregations(aggregations, self.config.aggregations, self.config[type].aggregations),
                     aggregationFrequency: self.util.setAggregationFrequency(aggregationFrequency, self.config.aggregationFrequency, self.config[type].aggregationFrequency),
-                    namespace: namespace || self.config.namespace
+                    namespace: namespace || self.config.namespace,
+                    sampleRate: sampleRate || self.config.sampleRate
                 };
             };
             this.mergeConfigs(clientConfig);
@@ -424,7 +431,7 @@
                     var time = this.measureTimeUserTiming(measureName);
                     if (time) {
                         // Push metrics to queue
-                        this.util.addItemToQueue("metrics", new this.metricsData(metricName, "timer", time, defaults.tags, defaults.aggregations, defaults.aggregationFrequency));
+                        this.util.addItemToQueue("metrics", new this.metricsData(metricName, "timer", time, defaults.tags, defaults.aggregations, defaults.aggregationFrequency, defaults.namespace, defaults.sampleRate));
                     } else {
                         logger.error("Failed to get measure time to register as timer value");
                     }
@@ -453,7 +460,7 @@
                 if (metricName && metricValue) {
                     options = options || {};
                     // Push metrics to queue
-                    var item = new this.metricsData(metricName, "timer", metricValue, options.tags, options.agg, options.aggFreq, options.namespace);
+                    var item = new this.metricsData(metricName, "timer", metricValue, options.tags, options.agg, options.aggFreq, options.namespace, options.sampleRate);
                     this.util.addItemToQueue("metrics", item);
                 } else {
                     logger.error("Undefined metric name/value to register as a timer");
@@ -475,7 +482,7 @@
                 if (metricName) {
                     options = options || {};
                     // Push metrics to queue
-                    var item = new this.metricsData(metricName, "counter", metricValue, options.tags, options.agg, options.aggFreq, options.namespace);
+                    var item = new this.metricsData(metricName, "counter", metricValue, options.tags, options.agg, options.aggFreq, options.namespace, options.sampleRate);
                     this.util.addItemToQueue("metrics", item);
                 } else {
                     logger.error("Undefined metric name to register as a counter");
@@ -496,7 +503,7 @@
                 if (metricName && metricValue) {
                     options = options || {};
                     // Push metrics to queue
-                    var item = new this.metricsData(metricName, "gauge", metricValue, options.tags, options.agg, options.aggFreq, options.namespace);
+                    var item = new this.metricsData(metricName, "gauge", metricValue, options.tags, options.agg, options.aggFreq, options.namespace, options.sampleRate);
                     this.util.addItemToQueue("metrics", item);
                 } else {
                     logger.error("Undefined metric name/value to register as a gauge");
diff --git a/dist/statful.min.js b/dist/statful.min.js
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "statful-client-javascript",
-  "version": "1.1.5",
+  "version": "1.1.6",
   "description": "Statful client for Javascript applications",
   "banner": "/**\n* <%= pkg.name %> <%= pkg.version %>\n* Copyright 2016 Statful <https://www.statful.com/>\n*/\n",
   "scripts": {
diff --git a/src/statful-util.js b/src/statful-util.js
@@ -121,8 +121,14 @@
          * @param {object} item - object to be sent
          */
         this.addItemToQueue = function (queueName, item) {
-            if (this.listQueues[queueName]) {
+            var sampleRateNormalized = (item.sampleRate || this.config.sampleRate || 100) / 100;
+
+            if (this.listQueues[queueName] && Math.random() <= sampleRateNormalized) {
                 this.listQueues[queueName].data.push(item);
+                return true;
+            } else {
+                logger.debug('Metric was discarded due to sample rate.');
+                return false;
             }
         };
 
diff --git a/src/statful.js b/src/statful.js
@@ -40,7 +40,8 @@
             aggregations: ['last']
         },
         timeout: 2000,
-        flushInterval: 10000
+        flushInterval: 10000,
+        sampleRate: 100
     };
 
     var logger;
@@ -76,15 +77,16 @@
                 });
             };
 
-            self.metricsData = function (name, type, value, tags, aggregations, aggregationFrequency, namespace) {
+            self.metricsData = function (name, type, value, tags, aggregations, aggregationFrequency, namespace, sampleRate) {
                 return {
                     name: name,
                     type: type,
                     value: value,
                     tags: self.util.setTags(tags || {}, self.config.tags, self.config[type].tags, self.config.app),
                     aggregations: self.util.setAggregations(aggregations, self.config.aggregations, self.config[type].aggregations),
                     aggregationFrequency: self.util.setAggregationFrequency(aggregationFrequency, self.config.aggregationFrequency, self.config[type].aggregationFrequency),
-                    namespace: namespace || self.config.namespace
+                    namespace: namespace || self.config.namespace,
+                    sampleRate: sampleRate || self.config.sampleRate
                 };
             };
 
@@ -229,7 +231,7 @@
                     if (time) {
                         // Push metrics to queue
                         this.util.addItemToQueue('metrics', new this.metricsData(metricName, 'timer', time,
-                            defaults.tags, defaults.aggregations, defaults.aggregationFrequency));
+                            defaults.tags, defaults.aggregations, defaults.aggregationFrequency, defaults.namespace, defaults.sampleRate));
                     } else {
                         logger.error('Failed to get measure time to register as timer value');
                     }
@@ -262,7 +264,7 @@
                     options = options || {};
 
                     // Push metrics to queue
-                    var item = new this.metricsData(metricName, 'timer', metricValue, options.tags, options.agg, options.aggFreq, options.namespace);
+                    var item = new this.metricsData(metricName, 'timer', metricValue, options.tags, options.agg, options.aggFreq, options.namespace, options.sampleRate);
                     this.util.addItemToQueue('metrics', item);
                 } else {
                     logger.error('Undefined metric name/value to register as a timer');
@@ -287,7 +289,7 @@
                     options = options || {};
 
                     // Push metrics to queue
-                    var item = new this.metricsData(metricName, 'counter', metricValue, options.tags, options.agg, options.aggFreq, options.namespace);
+                    var item = new this.metricsData(metricName, 'counter', metricValue, options.tags, options.agg, options.aggFreq, options.namespace, options.sampleRate);
                     this.util.addItemToQueue('metrics', item);
                 } else {
                     logger.error('Undefined metric name to register as a counter');
@@ -310,7 +312,7 @@
                     options = options || {};
 
                     // Push metrics to queue
-                    var item = new this.metricsData(metricName, 'gauge', metricValue, options.tags, options.agg, options.aggFreq, options.namespace);
+                    var item = new this.metricsData(metricName, 'gauge', metricValue, options.tags, options.agg, options.aggFreq, options.namespace, options.sampleRate);
                     this.util.addItemToQueue('metrics', item);
                 } else {
                     logger.error('Undefined metric name/value to register as a gauge');
diff --git a/tests/statful-util.tests.js b/tests/statful-util.tests.js
@@ -4,7 +4,8 @@ describe('Statful Util Unit testing', function () {
     beforeEach(function () {
         statfulUtil= new StatfulUtil({
             apiAddress: '//beacon.statful.com',
-            flushInterval: 30000
+            flushInterval: 30000,
+            sampleRate: 100
         });
     });
 
@@ -124,4 +125,33 @@ describe('Statful Util Unit testing', function () {
         expect(statfulUtil.listQueues.length).toEqual(0);
     });
 
+    it('should not add items to queue (sample rate)', function () {
+        expect(statfulUtil.listQueues.length).toEqual(0);
+
+        expect(statfulUtil.registerQueue('metrics', 'endpoint', 5000)).toEqual(true);
+
+        spyOn(Math, 'random').and.returnValue(0.5);
+
+        expect(statfulUtil.addItemToQueue('metrics', {
+            name: 'test',
+            type: 'counter',
+            value: 1,
+            sampleRate: 40
+        })).toBeFalsy();
+    });
+
+    it('should add items to queue (sample rate)', function () {
+        expect(statfulUtil.listQueues.length).toEqual(0);
+
+        expect(statfulUtil.registerQueue('metrics', 'endpoint', 5000)).toEqual(true);
+
+        spyOn(Math, 'random').and.returnValue(0.5);
+
+        expect(statfulUtil.addItemToQueue('metrics', {
+            name: 'test',
+            type: 'counter',
+            value: 1,
+            sampleRate: 60
+        })).toBeTruthy();
+    });
 });
diff --git a/tests/statful.tests.js b/tests/statful.tests.js
@@ -61,7 +61,8 @@ describe('Statful Client Unit testing', function () {
             tags: {mark: 'gauge', meh: 'yep'},
             aggregations: ['last'],
             aggregationFrequency: 300,
-            namespace: 'web'
+            namespace: 'web',
+            sampleRate: 100
         });
     });
 
@@ -91,7 +92,40 @@ describe('Statful Client Unit testing', function () {
             tags: {mark: 'gauge', meh: 'yep'},
             aggregations: ['last'],
             aggregationFrequency: 60,
-            namespace: 'web'
+            namespace: 'web',
+            sampleRate: 100
+        });
+    });
+
+    it('should override sample rate by type over global', function() {
+        statful.initialize({
+            tags: {},
+            aggregations: ['last'],
+            timer: {tags: {foo: 'bar'}},
+            gauge: {tags: {meh: 'yep'}, aggregations: [], aggregationFrequency: 60}
+        });
+
+        var util = statful.util;
+
+        spyOn(util, 'addItemToQueue');
+
+        var options = {
+            tags: {mark: 'gauge'},
+            agg: [],
+            sampleRate: 50
+        };
+
+        statful.gauge('test', 1234, options);
+
+        expect(util.addItemToQueue).toHaveBeenCalledWith('metrics', {
+            name: 'test',
+            type: 'gauge',
+            value:  1234,
+            tags: {mark: 'gauge', meh: 'yep'},
+            aggregations: ['last'],
+            aggregationFrequency: 60,
+            namespace: 'web',
+            sampleRate: 50
         });
     });
 
@@ -122,7 +156,8 @@ describe('Statful Client Unit testing', function () {
             tags: {mark: 'gauge', meh: 'yep'},
             aggregations: ['last'],
             aggregationFrequency: 300,
-            namespace: 'web'
+            namespace: 'web',
+            sampleRate: 100
         });
     });
 
@@ -151,7 +186,8 @@ describe('Statful Client Unit testing', function () {
             tags: {mark: 'gauge', meh: 'yep'},
             aggregations: [],
             aggregationFrequency: 10,
-            namespace: 'web'
+            namespace: 'web',
+            sampleRate: 100
         });
     });
 
@@ -301,7 +337,8 @@ describe('Statful Client Unit testing', function () {
             tags: {mark: 'measure', unit: 'ms'},
             aggregations: ['avg', 'p90', 'count'],
             aggregationFrequency: 10,
-            namespace: 'web'
+            namespace: 'web',
+            sampleRate: 100
         });
     });
 
@@ -398,7 +435,8 @@ describe('Statful Client Unit testing', function () {
             tags: {mark: 'gauge'},
             aggregations: ['last'],
             aggregationFrequency: 30,
-            namespace: 'web'
+            namespace: 'web',
+            sampleRate: 100
         });
     });
 });

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "statful-client-javascript",`
`3`		`- "version": "1.1.5",`
	`3`	`+ "version": "1.1.6",`
`4`	`4`	`"description": "Statful client for Javascript applications",`
`5`	`5`	`"main": "dist/statful.js",`
`6`	`6`	`"authors": [`