From fe0b354a0d085c6c73e2710d048b1fc602e350c5 Mon Sep 17 00:00:00 2001 From: Chopin Ngo Date: Thu, 5 Jan 2017 17:48:44 +0800 Subject: [PATCH 1/7] Add disabledWebinterface option --- .gitignore | 3 ++- bin.js | 3 ++- lib/requestHandler.js | 10 +++------- proxy.js | 6 ++++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 2fa5ecdff..fbdd99ca3 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,5 @@ coverage build/Release node_modules .lock-wscript -temp \ No newline at end of file +temp +tags diff --git a/bin.js b/bin.js index df8c74753..b9229ac3a 100755 --- a/bin.js +++ b/bin.js @@ -15,6 +15,7 @@ program .option('-t, --type [value]', 'http|https, http for default') .option('-p, --port [value]', 'proxy port, 8001 for default') .option('-w, --web [value]' , 'web GUI port, 8002 for default') + .option('-I, --no-webinterface', 'disable WebInterface') .option('-f, --file [value]', 'save request data to a specified file, will use in-memory db if not specified') .option('-r, --rule [value]', 'path for rule file,') .option('-g, --root [value]', 'generate root CA') @@ -91,7 +92,7 @@ if(program.clear){ throttle : program.throttle, webPort : program.web, rule : ruleModule, - disableWebInterface : false, + disableWebInterface : !program.webinterface, setAsGlobalProxy : program.global, interceptHttps : program.intercept, silent : program.silent diff --git a/lib/requestHandler.js b/lib/requestHandler.js index 04b0cda4c..48edf64a6 100644 --- a/lib/requestHandler.js +++ b/lib/requestHandler.js @@ -49,9 +49,7 @@ function userRequestHandler(req,userRes){ req : req, startTime : new Date().getTime() }; - if(global.recorder){ - resourceInfoId = global.recorder.appendRecord(resourceInfo); - } + resourceInfoId = global.recorder ? global.recorder.appendRecord(resourceInfo) : -1; logUtil.printLog(color.green("\nreceived request to : " + host + path)); @@ -61,9 +59,7 @@ function userRequestHandler(req,userRes){ routeReq ],function(){ //mark some ext info - if(req.anyproxy_map_local){ - global.recorder.updateExtInfo(resourceInfoId, {map : req.anyproxy_map_local}); - } + req.anyproxy_map_local && global.recorder && global.recorder.updateExtInfo(resourceInfoId, {map : req.anyproxy_map_local}); }); //get request body @@ -297,7 +293,7 @@ function connectReqHandler(req, socket, head){ req : req, startTime : new Date().getTime() }; - resourceInfoId = global.recorder.appendRecord(resourceInfo); + resourceInfoId = global.recorder ? global.recorder.appendRecord(resourceInfo) : -1; var proxyPort, proxyHost, diff --git a/proxy.js b/proxy.js index 8f566d7f7..cca2ba239 100644 --- a/proxy.js +++ b/proxy.js @@ -153,8 +153,10 @@ function proxyServer(option){ //start web socket service function(callback){ - self.ws = new wsServer({port : socketPort}); - callback(null); + if (!disableWebInterface) { + self.ws = new wsServer({port : socketPort}); + callback(null); + } }, //start web interface From 37ee345def0495bdb1ffe1f391158a32077d82d7 Mon Sep 17 00:00:00 2001 From: Chopin Ngo Date: Fri, 6 Jan 2017 09:50:46 +0800 Subject: [PATCH 2/7] Add disablePersistence option --- bin.js | 4 +++- proxy.js | 13 ++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/bin.js b/bin.js index b9229ac3a..4f03b8357 100755 --- a/bin.js +++ b/bin.js @@ -16,6 +16,7 @@ program .option('-p, --port [value]', 'proxy port, 8001 for default') .option('-w, --web [value]' , 'web GUI port, 8002 for default') .option('-I, --no-webinterface', 'disable WebInterface') + .option('-P, --no-persistence', 'disable data persistence, will also disable WebInterface') .option('-f, --file [value]', 'save request data to a specified file, will use in-memory db if not specified') .option('-r, --rule [value]', 'path for rule file,') .option('-g, --root [value]', 'generate root CA') @@ -92,7 +93,8 @@ if(program.clear){ throttle : program.throttle, webPort : program.web, rule : ruleModule, - disableWebInterface : !program.webinterface, + disableWebInterface : !program.persistence || !program.webinterface, + disablePersistence : !program.persistence, setAsGlobalProxy : program.global, interceptHttps : program.intercept, silent : program.silent diff --git a/proxy.js b/proxy.js index cca2ba239..cd77e526b 100644 --- a/proxy.js +++ b/proxy.js @@ -63,7 +63,8 @@ function proxyServer(option){ proxyWebPort = option.webPort || DEFAULT_WEB_PORT, //port for web interface socketPort = option.socketPort || DEFAULT_WEBSOCKET_PORT, //port for websocket proxyConfigPort = option.webConfigPort || DEFAULT_CONFIG_PORT, //port to ui config server - disableWebInterface = !!option.disableWebInterface, + disableWebInterface = !!option.disablePersistence || !!option.disableWebInterface, + disablePersistence = !!option.disablePersistence, ifSilent = !!option.silent; if(ifSilent){ @@ -110,10 +111,12 @@ function proxyServer(option){ //clear cache dir, prepare recorder function(callback){ util.clearCacheDir(function(){ - if(option.dbFile){ - global.recorder = new Recorder({filename: option.dbFile}); - }else{ - global.recorder = new Recorder(); + if (!option.disablePersistence) { + if(option.dbFile){ + global.recorder = new Recorder({filename: option.dbFile}); + }else{ + global.recorder = new Recorder(); + } } callback(); }); From f6e945fd156c8fd5a988298d52169699cfcacd8b Mon Sep 17 00:00:00 2001 From: Chopin Ngo Date: Wed, 22 Feb 2017 16:48:56 +0800 Subject: [PATCH 3/7] Fix annotation misspell --- proxy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy.js b/proxy.js index cd77e526b..5e5d77e9c 100644 --- a/proxy.js +++ b/proxy.js @@ -122,7 +122,7 @@ function proxyServer(option){ }); }, - //creat proxy server + //create proxy server function(callback){ if(proxyType == T_TYPE_HTTPS){ certMgr.getCertificate(proxyHost,function(err,keyContent,crtContent){ From 3eec3255ddfc61b236e058c2ee6ceabbcf3f1354 Mon Sep 17 00:00:00 2001 From: Chopin Ngo Date: Wed, 22 Feb 2017 22:53:06 +0800 Subject: [PATCH 4/7] Add proxy auth support --- .gitignore | 1 + bin.js | 66 +++++++++++++++++++++++++++++++++ lib/requestHandler.js | 85 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 151 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fbdd99ca3..b5bdc5342 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ node_modules .lock-wscript temp tags +auth.db diff --git a/bin.js b/bin.js index 4f03b8357..4dcc04bba 100755 --- a/bin.js +++ b/bin.js @@ -25,9 +25,16 @@ program .option('-s, --silent', 'do not print anything into terminal') .option('-c, --clear', 'clear all the tmp certificates') .option('-o, --global', 'set as global proxy for system') + .option('-A, --auth', 'enable proxy authorization') + .option('-a, --adduser [ ]...', 'add proxy user') + .option('-m, --moduser ', 'modify proxy user password') + .option('-d, --deluser [user]...', 'delete proxy user') + .option('-F, --authFile [value]', 'save proxy auth data to a specified file, if not specified use __dirname/auth.db') .option('install', '[alpha] install node modules') .parse(process.argv); +var authFile = program.authFile ? path.resolve(process.cwd(), program.authFile) : path.resolve(__dirname, 'auth.db'); + if(program.clear){ require("./lib/certMgr").clearCerts(function(){ console.log( color.green("all certs cleared") ); @@ -48,6 +55,58 @@ if(program.clear){ }); npm.registry.log.on("log", function (message) {}); }); +}else if (program.adduser || program.deluser || program.moduser){ + var db, Datastore = require('nedb'), + args = program.args, + users = []; + + try { + db = new Datastore({filename: authFile, autoload: true}); + db.ensureIndex({ fieldName: 'username', unique: true}); + db.persistence.setAutocompactionInterval(5001); + + console.log("proxy auth file : " + authFile); + } catch (e) { + console.log('create proxy auth database file failed, ' + e); + process.exit(-1); + } + + if (program.adduser) { + args.unshift(program.adduser); + + if (args.length % 2 !== 0) { + console.log('add user failed, every user must be set a password.'); + process.exit(-1); + } + + for (var i = 0; i < args.length; i += 2) { + users.push({username: args[i], password: args[i + 1]}); + } + + db.insert(users, function (err) { + err && console.log('create proxy user failed: ' + err.message); + }); + } else if (program.deluser) { + args.push(program.deluser); + + args.forEach(function (user) { + db.remove({username: user}, {}, function (err) { + err && console.log('delete user failed: ' + err.message); + }); + }); + } else { + if (args.length <= 0) { + console.log('user password must be set.'); + process.exit(-1); + } + + db.update({username: program.moduser}, {username: program.moduser, password: args[0]}, {upsert: true}, function (err) { + err && console.log('modify proxy user failed: ' + err.message); + }); + } + + // db.persistence.compactDatafile(); + db.persistence.stopAutocompaction(); }else{ var proxy = require("./proxy.js"); var ruleModule; @@ -56,6 +115,13 @@ if(program.clear){ logUtil.setPrintStatus(false); } + if (program.auth) { + var Datastore = require('nedb'); + global.auth = new Datastore({filename: authFile, autoload: true}); + global.auth.persistence.setAutocompactionInterval(5001); + logUtil.printLog('proxy auth file loaded : ' + authFile); + } + if(program.rule){ var ruleFilePath = path.resolve(process.cwd(),program.rule); try{ diff --git a/lib/requestHandler.js b/lib/requestHandler.js index 48edf64a6..c0917ad6d 100644 --- a/lib/requestHandler.js +++ b/lib/requestHandler.js @@ -1,5 +1,5 @@ var http = require("http"), - https = require("https"), + https = require("https"), net = require("net"), fs = require("fs"), url = require("url"), @@ -51,6 +51,10 @@ function userRequestHandler(req,userRes){ }; resourceInfoId = global.recorder ? global.recorder.appendRecord(resourceInfo) : -1; + if (global.auth && protocol === 'http' && auth(req, userRes, resourceInfo, resourceInfoId) === false) { + return ; + } + logUtil.printLog(color.green("\nreceived request to : " + host + path)); //get request body and route to local or remote @@ -270,6 +274,10 @@ function connectReqHandler(req, socket, head){ resourceInfo, resourceInfoId; + if (global.auth && auth(req, socket) === false) { + return ; + } + var shouldIntercept = userRule.shouldInterceptHttpsReq(req); //bypass webSocket on webinterface @@ -424,6 +432,81 @@ function setRules(newRule){ } } +// proxy authorization +function auth(req, res, info = null, id = null) { + var index, header = req.headers["proxy-authorization"], + end = function (code) { + if (info === null) { + if (global.recorder) { + global.recorder.appendRecord({ + host : req.url.split(":")[0], + method : req.method, + path : "", + url : "https://" + host, + req : req, + startTime : new Date().getTime(), + endTime : new Date().getTime(), + statusCode: code, + resHeader : {}, + resBody : "", + length : 0 + }); + } + } else { + info.endTime = new Date().getTime(); + info.statusCode = code; + info.resHeader = {}; + info.resBody = ""; + info.length = 0; + + global.recorder && global.recorder.updateRecord(id, info); + } + + + if (res.writeHead) { + res.removeHeader('Date'); + res.writeHead(code, {"Connection": "closed", "Content-Length": 0}); + } else { + res.write('HTTP/' + req.httpVersion + ' ' + code + ' ' + + (code === 401 ? 'Unauthorized' : 'Proxy Authentication Required') + + '\r\n\r\n', 'UTF-8'); + } + + res.end(); + return false; + }; + + if (!header) { + return end(407); + } + + if (!header.startsWith("Basic ")) { + return end(401); + } + + header = Buffer.from(header.split(" ", 2)[1], "base64"); + + if (header.length <= 0) { + return end(401); + } + + header = header.toString("ascii"); + index = header.indexOf(":"); + + if (index === -1) { + return end(401); + } + + global.auth.findOne({username: header.slice(0, index++), password: header.slice(index)}, function (err, doc) { + if (err || doc === null) { + end(401); + index = -1; + } + }); + + return index !== -1; +} + function getRuleSummary(){ return userRule.summary(); } From f758e8d638eca76b144181924280731313159459 Mon Sep 17 00:00:00 2001 From: Chopin Ngo Date: Thu, 23 Feb 2017 12:07:11 +0800 Subject: [PATCH 5/7] Fix not blocked request if proxy auth failed Add proxy auth unit tests --- bin.js | 11 ++-- lib/requestHandler.js | 59 +++++++-------------- proxy.js | 9 ++++ test/proxy_auth_spec.js | 100 +++++++++++++++++++++++++++++++++++ test/util/ProxyServerUtil.js | 16 +++++- 5 files changed, 146 insertions(+), 49 deletions(-) create mode 100644 test/proxy_auth_spec.js diff --git a/bin.js b/bin.js index 4dcc04bba..9270593e8 100755 --- a/bin.js +++ b/bin.js @@ -115,13 +115,6 @@ if(program.clear){ logUtil.setPrintStatus(false); } - if (program.auth) { - var Datastore = require('nedb'); - global.auth = new Datastore({filename: authFile, autoload: true}); - global.auth.persistence.setAutocompactionInterval(5001); - logUtil.printLog('proxy auth file loaded : ' + authFile); - } - if(program.rule){ var ruleFilePath = path.resolve(process.cwd(),program.rule); try{ @@ -163,6 +156,8 @@ if(program.clear){ disablePersistence : !program.persistence, setAsGlobalProxy : program.global, interceptHttps : program.intercept, - silent : program.silent + silent : program.silent, + auth : program.auth, + authFile : authFile }); } diff --git a/lib/requestHandler.js b/lib/requestHandler.js index c0917ad6d..361d66246 100644 --- a/lib/requestHandler.js +++ b/lib/requestHandler.js @@ -51,14 +51,13 @@ function userRequestHandler(req,userRes){ }; resourceInfoId = global.recorder ? global.recorder.appendRecord(resourceInfo) : -1; - if (global.auth && protocol === 'http' && auth(req, userRes, resourceInfo, resourceInfoId) === false) { - return ; - } - logUtil.printLog(color.green("\nreceived request to : " + host + path)); //get request body and route to local or remote async.series([ + function (callback) { + auth(callback, req, userRes, resourceInfo, resourceInfoId, protocol); + }, fetchReqData, routeReq ],function(){ @@ -69,6 +68,7 @@ function userRequestHandler(req,userRes){ //get request body function fetchReqData(callback){ var postData = []; + req.on("data",function(chunk){ postData.push(chunk); }); @@ -274,10 +274,6 @@ function connectReqHandler(req, socket, head){ resourceInfo, resourceInfoId; - if (global.auth && auth(req, socket) === false) { - return ; - } - var shouldIntercept = userRule.shouldInterceptHttpsReq(req); //bypass webSocket on webinterface @@ -309,7 +305,9 @@ function connectReqHandler(req, socket, head){ httpsServerMgrInstance; async.series([ - + function (callback) { + auth(callback, req, socket, resourceInfo, resourceInfoId, 'http'); + }, //check if internal https server exists function(callback){ if(!shouldIntercept){ @@ -433,35 +431,20 @@ function setRules(newRule){ } // proxy authorization -function auth(req, res, info = null, id = null) { +function auth(cb, req, res, info, id, protocol) { + if (protocol === 'https' || !global.auth) { + return cb(); + } + var index, header = req.headers["proxy-authorization"], end = function (code) { - if (info === null) { - if (global.recorder) { - global.recorder.appendRecord({ - host : req.url.split(":")[0], - method : req.method, - path : "", - url : "https://" + host, - req : req, - startTime : new Date().getTime(), - endTime : new Date().getTime(), - statusCode: code, - resHeader : {}, - resBody : "", - length : 0 - }); - } - } else { - info.endTime = new Date().getTime(); - info.statusCode = code; - info.resHeader = {}; - info.resBody = ""; - info.length = 0; - - global.recorder && global.recorder.updateRecord(id, info); - } + info.endTime = new Date().getTime(); + info.statusCode = code; + info.resHeader = {}; + info.resBody = ""; + info.length = 0; + global.recorder && global.recorder.updateRecord(id, info); if (res.writeHead) { res.removeHeader('Date'); @@ -499,12 +482,10 @@ function auth(req, res, info = null, id = null) { global.auth.findOne({username: header.slice(0, index++), password: header.slice(index)}, function (err, doc) { if (err || doc === null) { - end(401); - index = -1; + return end(401); } + cb(); }); - - return index !== -1; } function getRuleSummary(){ diff --git a/proxy.js b/proxy.js index 5e5d77e9c..a3257164c 100644 --- a/proxy.js +++ b/proxy.js @@ -52,6 +52,8 @@ var requestHandler = util.freshRequire('./requestHandler'); //option.disableWebInterface //option.silent : false(default) //option.interceptHttps ,internal param for https +//option.auth : false(default) +//option.authFile : __dirname/auth.db(default) function proxyServer(option){ option = option || {}; @@ -71,6 +73,13 @@ function proxyServer(option){ logUtil.setPrintStatus(false); } + if (option.auth) { + var Datastore = require('nedb'); + global.auth = new Datastore({filename: option.authFile, autoload: true}); + global.auth.persistence.setAutocompactionInterval(5001); + logUtil.printLog('proxy auth file loaded : ' + option.authFile); + } + // copy the rule to keep the original proxyRules independent proxyRules = Object.assign({}, proxyRules); diff --git a/test/proxy_auth_spec.js b/test/proxy_auth_spec.js new file mode 100644 index 000000000..f5410c98e --- /dev/null +++ b/test/proxy_auth_spec.js @@ -0,0 +1,100 @@ +/* +* test for proxy auth +* +*/ + +const ProxyServerUtil = require('./util/ProxyServerUtil.js'); +const { proxyGet, generateUrl } = require('./util/HttpUtil.js'); +const Server = require('./server/server.js'); +const { printLog } = require('./util/CommonUtil.js'); + +testAuth('http'); +testAuth('https'); + +function testAuth(protocol) { + describe('Proxy auth should be working in :' + protocol, () => { + let proxyServer ; + let serverInstance ; + + beforeAll((done) => { + jasmine.DEFAULT_TIMEOUT_INTERVAL = 2000000; + printLog('Start server for proxy_auth_spec'); + + global.auth = null; + serverInstance = new Server(); + + proxyServer = ProxyServerUtil.proxyServerWithProxyAuth(); + + setTimeout(function() { + done(); + }, 2000); + }); + + afterAll(() => { + serverInstance && serverInstance.close(); + proxyServer && proxyServer.close(); + global.auth = null; + printLog('Close server for proxy_auth_spec'); + + }); + + it('Should be return 407 if not set proxy auth', done => { + const url = generateUrl(protocol, '/test'); + proxyGet(url, {}) + .then(res => { + if (protocol === 'http') { + expect(res.statusCode).toEqual(407); + expect(res.body).toEqual(''); + done(); + } else { + console.log('error happened in proxy get for proxy auth without auth'); + done.fail('error happened when test proxy auth without auth'); + } + }, error => { + if (protocol === 'http') { + console.log('error happened in proxy get for proxy auth without auth: ', error); + done.fail('error happened when test proxy auth without auth'); + } else { + expect(error.message.slice(-3)).toEqual('407'); + done(); + } + }); + }); + + it('Should be return 401 if proxy auth failed', done => { + const url = generateUrl(protocol, '/test'); + proxyGet(url, {}, {'Proxy-Authorization': 'Basic Y2hvcGluOnBhc3M='}) + .then(res => { + if (protocol === 'http') { + expect(res.statusCode).toEqual(401); + expect(res.body).toEqual(''); + done(); + } else { + console.log('error happened in proxy get for proxy auth with error pass'); + done.fail('error happened when test proxy auth with error pass'); + } + }, error => { + if (protocol === 'http') { + console.log('error happened in proxy get for proxy auth with error pass: ', error); + done.fail('error happened when test proxy auth with error pass'); + } else { + expect(error.message.slice(-3)).toEqual('401'); + done(); + } + }); + }); + + it('Should be normal if proxy auth success', done => { + const url = generateUrl(protocol, '/test'); + proxyGet(url, {}, {'Proxy-Authorization': 'Basic Y2hvcGluOm5nbw=='}) + .then(res => { + expect(res.statusCode).toEqual(200); + expect(res.body).toEqual('something'); + done(); + }, error => { + console.log('error happened in proxy get for proxy auth: ', error); + done.fail('error happened when test proxy auth'); + }); + }); + }); +} diff --git a/test/util/ProxyServerUtil.js b/test/util/ProxyServerUtil.js index 0a333ec93..d118eed90 100644 --- a/test/util/ProxyServerUtil.js +++ b/test/util/ProxyServerUtil.js @@ -5,6 +5,7 @@ let proxy = require('../../proxy.js'); const util = require('../../lib/util.js'); +const path = require('path'); const DEFAULT_OPTIONS = { type: "http", @@ -58,8 +59,19 @@ function proxyServerWithoutHttpsIntercept (rule) { return new proxy.proxyServer(options); } +function proxyServerWithProxyAuth () { + proxy = util.freshRequire('../proxy.js'); + + const options = util.merge({}, DEFAULT_OPTIONS); + options.auth = true; + options.authFile = path.resolve(process.cwd(), 'test/data/auth.db'); + + return new proxy.proxyServer(options); +} + module.exports = { defaultProxyServer, proxyServerWithoutHttpsIntercept, - proxyServerWithRule -}; \ No newline at end of file + proxyServerWithRule, + proxyServerWithProxyAuth +}; From bba805b1d3eeea4f3eae7f18d2a1c1345298157b Mon Sep 17 00:00:00 2001 From: Chopin Ngo Date: Thu, 23 Feb 2017 12:17:45 +0800 Subject: [PATCH 6/7] Fix ignored proxy auth unit tests data file --- test/data/.gitignore | 1 + test/data/auth.db | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 test/data/.gitignore create mode 100644 test/data/auth.db diff --git a/test/data/.gitignore b/test/data/.gitignore new file mode 100644 index 000000000..f9be8dfe0 --- /dev/null +++ b/test/data/.gitignore @@ -0,0 +1 @@ +!* diff --git a/test/data/auth.db b/test/data/auth.db new file mode 100644 index 000000000..a466bf732 --- /dev/null +++ b/test/data/auth.db @@ -0,0 +1,2 @@ +{"username":"chopin","password":"ngo","_id":"WkKABWbVTmecX1Ee"} +{"$$indexCreated":{"fieldName":"username","unique":true,"sparse":false}} From ed6ebd74c881db91acfdb5ae3a709b97280fa860 Mon Sep 17 00:00:00 2001 From: Chopin Ngo Date: Thu, 23 Feb 2017 12:53:24 +0800 Subject: [PATCH 7/7] Add sample rule of remove request proxy auth header --- rule_sample/rule_remove_proxy_auth_header.js | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 rule_sample/rule_remove_proxy_auth_header.js diff --git a/rule_sample/rule_remove_proxy_auth_header.js b/rule_sample/rule_remove_proxy_auth_header.js new file mode 100644 index 000000000..25f424da6 --- /dev/null +++ b/rule_sample/rule_remove_proxy_auth_header.js @@ -0,0 +1,11 @@ +//rule scheme : + +module.exports = { + replaceRequestOption : function(req,option){ + var newOption = option; + delete newOption.headers['proxy-authorization']; + delete newOption.headers['proxy-connection']; + + return newOption; + } +};