From d1ab3f2d3a155aa6d545e2474f445e8c000556c4 Mon Sep 17 00:00:00 2001 From: Prasun Date: Tue, 22 Apr 2014 12:49:39 +0530 Subject: [PATCH 1/5] allow response encoding to be null --- lib/requestify.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/requestify.js b/lib/requestify.js index 9293f1b..5736af6 100644 --- a/lib/requestify.js +++ b/lib/requestify.js @@ -187,7 +187,7 @@ var Requestify = (function() { * @returns {string|Requestify} */ responseEncoding: function(value) { - if (!value) { + if (value === undefined) { return responseEncoding; } From 91bc742013f78b614622855abcd9be163a92eadf Mon Sep 17 00:00:00 2001 From: Prasun Date: Tue, 22 Apr 2014 20:07:10 +0530 Subject: [PATCH 2/5] Adding buffered response, which is useful is response is gzipped --- lib/requestify.js | 29 +++++++++++++++++++++++++++-- lib/response.js | 30 ++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/lib/requestify.js b/lib/requestify.js index 5736af6..eefaebf 100644 --- a/lib/requestify.js +++ b/lib/requestify.js @@ -8,6 +8,8 @@ var http = require('http'), cache = require('./cache'), https = require('https'), logger = require('./logger'), + /*content-types that requires buffer encoding*/ + BUFFER_RESPONSE_TYPES = ['gzip', 'deflate'], redisTransporter = require('./cache-transporters/redis-transporter'); var Requestify = (function() { @@ -69,6 +71,19 @@ var Requestify = (function() { }); } + /** + * Based on response content type, returns true if Buffered response is needed + * @param {ttp.ServerResponse} res + **/ + function isBufferResponse(res){ + isBufferResponse.__requiresResponseBuffer = + isBufferResponse.__requiresResponseBuffer === undefined ? + (BUFFER_RESPONSE_TYPES.indexOf(res.headers['content-encoding']) !== -1) : + isBufferResponse.__requiresResponseBuffer; + + return isBufferResponse.__requiresResponseBuffer; + } + /** * Executes the given request object. * @param {Request} request @@ -97,12 +112,22 @@ var Requestify = (function() { clearTimeout(timeout); var response = new Response(res.statusCode, res.headers); - res.setEncoding(responseEncoding); + + if(!isBufferResponse(res)){ + res.setEncoding(responseEncoding); + } + res.on('data', function(chunk) { - response.setChunk(chunk); + if(isBufferResponse(res)){ + response.setResponseChunks(chunk); + } + response.setChunk(chunk); }); res.on('end', function() { + if(isBufferResponse(res)){ + response.finalizeBuffer(); + } if (isSuccessful(response.code)) { storeCache(request.getFullUrl(), response.getCode(), response.getHeaders(), response.body, request.cache); defer.resolve(response); diff --git a/lib/response.js b/lib/response.js index 2592d4c..72762a1 100644 --- a/lib/response.js +++ b/lib/response.js @@ -3,6 +3,7 @@ var url = require('url'), queryString = require('querystring'), _ = require('underscore'), + responseChunks = [], $ = require('jquery'); /** @@ -16,6 +17,7 @@ function Response(code, headers, body) { this.code = code; this.headers = headers; this.body = body || ''; + this.bufferedResponse; } /** @@ -29,12 +31,35 @@ Response.prototype.setChunk = function(chunk) { return this; }; +/** + * Sets one response body chunk to the response body + * @param {buffer} chunk + * @returns {Response} + */ +Response.prototype.setResponseChunks = function(chunk) { + responseChunks.push(chunk); + console.log('chunkLength:' + chunk.length); + return this; +}; + +/** +* +*/ +Response.prototype.finalizeBuffer = function(chunk) { + //console.log('totalBufLength:' + totalBufLength); + //console.dir(responseChunks); + this.bufferedResponse = Buffer.concat(responseChunks); + return this; +}; + + /** * Returns the decoded || raw string of the request * @returns {string|object} */ Response.prototype.getBody = function() { - var responseType = this.getHeader('content-type'); + var responseType = this.getHeader('content-type'), + zlib; if (!responseType) { return this.body; @@ -47,7 +72,7 @@ Response.prototype.getBody = function() { console.log('Failed parsing expected JSON response, returned raw response'); return this.body; } - } + } if (responseType.indexOf('application/x-www-form-urlencoded') !== -1) { return queryString.parse(this.body); @@ -57,6 +82,7 @@ Response.prototype.getBody = function() { return $(this.body); } + console.log('body done return'); return this.body; }; From b11e7d5e4707b2ee36a3ea9d09462606c5233a20 Mon Sep 17 00:00:00 2001 From: Prasun Date: Wed, 23 Apr 2014 15:30:28 +0530 Subject: [PATCH 3/5] remove caching of buffer flag, it may screw concurrent requests --- lib/requestify.js | 13 ++++--------- lib/response.js | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/lib/requestify.js b/lib/requestify.js index eefaebf..b0e29ab 100644 --- a/lib/requestify.js +++ b/lib/requestify.js @@ -75,13 +75,8 @@ var Requestify = (function() { * Based on response content type, returns true if Buffered response is needed * @param {ttp.ServerResponse} res **/ - function isBufferResponse(res){ - isBufferResponse.__requiresResponseBuffer = - isBufferResponse.__requiresResponseBuffer === undefined ? - (BUFFER_RESPONSE_TYPES.indexOf(res.headers['content-encoding']) !== -1) : - isBufferResponse.__requiresResponseBuffer; - - return isBufferResponse.__requiresResponseBuffer; + function isBufferResponse(res){ + return BUFFER_RESPONSE_TYPES.indexOf(res.headers['content-encoding']) !== -1; } /** @@ -126,8 +121,8 @@ var Requestify = (function() { res.on('end', function() { if(isBufferResponse(res)){ - response.finalizeBuffer(); - } + response.finalizeBuffer(); + } if (isSuccessful(response.code)) { storeCache(request.getFullUrl(), response.getCode(), response.getHeaders(), response.body, request.cache); defer.resolve(response); diff --git a/lib/response.js b/lib/response.js index 72762a1..97713c4 100644 --- a/lib/response.js +++ b/lib/response.js @@ -27,7 +27,7 @@ function Response(code, headers, body) { */ Response.prototype.setChunk = function(chunk) { this.body += chunk; - + console.log('chunl set'); return this; }; From bb7ce1331a1f51df23c9a0e6638ad70686d82e68 Mon Sep 17 00:00:00 2001 From: Prasun Date: Wed, 23 Apr 2014 15:47:38 +0530 Subject: [PATCH 4/5] removed log statements --- lib/response.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/response.js b/lib/response.js index 97713c4..5c0fe71 100644 --- a/lib/response.js +++ b/lib/response.js @@ -27,7 +27,6 @@ function Response(code, headers, body) { */ Response.prototype.setChunk = function(chunk) { this.body += chunk; - console.log('chunl set'); return this; }; @@ -38,7 +37,6 @@ Response.prototype.setChunk = function(chunk) { */ Response.prototype.setResponseChunks = function(chunk) { responseChunks.push(chunk); - console.log('chunkLength:' + chunk.length); return this; }; @@ -82,7 +80,6 @@ Response.prototype.getBody = function() { return $(this.body); } - console.log('body done return'); return this.body; }; From 1ed3e9d296a0cb230acf99c2c4197f603bdbf8ac Mon Sep 17 00:00:00 2001 From: Prasun Date: Thu, 24 Apr 2014 14:32:35 +0530 Subject: [PATCH 5/5] Buffer Response Chunks array as this property for concurrent usage --- lib/response.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/response.js b/lib/response.js index 5c0fe71..5b4a04a 100644 --- a/lib/response.js +++ b/lib/response.js @@ -3,7 +3,6 @@ var url = require('url'), queryString = require('querystring'), _ = require('underscore'), - responseChunks = [], $ = require('jquery'); /** @@ -14,6 +13,7 @@ var url = require('url'), * @constructor */ function Response(code, headers, body) { + this.responseChunks = []; this.code = code; this.headers = headers; this.body = body || ''; @@ -36,7 +36,7 @@ Response.prototype.setChunk = function(chunk) { * @returns {Response} */ Response.prototype.setResponseChunks = function(chunk) { - responseChunks.push(chunk); + this.responseChunks.push(chunk); return this; }; @@ -46,7 +46,7 @@ Response.prototype.setResponseChunks = function(chunk) { Response.prototype.finalizeBuffer = function(chunk) { //console.log('totalBufLength:' + totalBufLength); //console.dir(responseChunks); - this.bufferedResponse = Buffer.concat(responseChunks); + this.bufferedResponse = Buffer.concat(this.responseChunks); return this; };