From daf3dc5d7f19a44f601a8c40dab5b5ebafd8c889 Mon Sep 17 00:00:00 2001 From: Mark van der Walle Date: Mon, 9 May 2016 15:21:33 +0200 Subject: [PATCH 1/2] Adds support for gzip and deflate compression --- lib/twitter.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/twitter.js b/lib/twitter.js index 757ab18..60e8d81 100644 --- a/lib/twitter.js +++ b/lib/twitter.js @@ -2,6 +2,7 @@ var request = require('request') , split = require('split') , Writable = require('stream').Writable , util = require('util') + , zlib = require('zlib') function backoff (current, max, step, _value) { return function () { @@ -209,6 +210,7 @@ Twitter.prototype.connect = function () { this.stream = request.post({ url: this.twitterUrl, oauth: this.oauth, + gzip: true, form: { track: Object.keys(this._filters[FILTER_TYPE_TRACKING]).join(','), locations: Object.keys(this._filters[FILTER_TYPE_LOCATION]).join(','), @@ -252,8 +254,18 @@ Twitter.prototype.connect = function () { } catch (e) {} }) - this.parser = res.pipe(this.parser, {end: false}) - this.parser.pipe(this) + var encoding = res.headers['content-encoding'] + if (encoding === 'gzip') { + console.log('initializing gunzip'); + this.parser = res.pipe(zlib.createGunzip()).pipe(this.parser, {end: false}) + this.parser.pipe(this) + } else if (encoding === 'deflate') { + this.parser = res.pipe(zlib.createInflate()).pipe(this.parser, {end: false}) + this.parser.pipe(this) + } else { + this.parser = res.pipe(this.parser, {end: false}) + this.parser.pipe(this) + } // Handle this: https://dev.twitter.com/docs/streaming-apis/connecting#Stalls // Abort the connection and reconnect if we haven't received an update for 90 seconds From 80296a0b4cbf988c944bbc7053bad6d906d57996 Mon Sep 17 00:00:00 2001 From: Mark van der Walle Date: Wed, 11 May 2016 14:05:06 +0200 Subject: [PATCH 2/2] Make gzip optional and default to false --- lib/twitter.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/twitter.js b/lib/twitter.js index 60e8d81..0c784b4 100644 --- a/lib/twitter.js +++ b/lib/twitter.js @@ -20,9 +20,9 @@ var FILTER_TYPE_TRACKING = 'tracking' , FILTER_TYPE_FOLLOW = 'follow' , FILTER_TYPE_LANGUAGE = 'language' -var Twitter = function (oauth) { +var Twitter = function (oauth, options) { if(!(this instanceof Twitter)) { - return new Twitter(oauth) + return new Twitter(oauth, options) } if (!oauth || !oauth.consumer_secret || !oauth.consumer_key || !oauth.token || !oauth.token_secret) { @@ -30,6 +30,9 @@ var Twitter = function (oauth) { } this.oauth = oauth + options.gzip = options.gzip ? options.gzip : false; + this.options = options; + this._filters = { tracking: {}, location: {}, @@ -210,7 +213,7 @@ Twitter.prototype.connect = function () { this.stream = request.post({ url: this.twitterUrl, oauth: this.oauth, - gzip: true, + gzip: this.options.gzip, form: { track: Object.keys(this._filters[FILTER_TYPE_TRACKING]).join(','), locations: Object.keys(this._filters[FILTER_TYPE_LOCATION]).join(','),