From 827cf71c0fdd61d94dc92fadf2fe6889093471b7 Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Mon, 7 Aug 2017 14:47:19 +0200 Subject: [PATCH 01/14] Added new model SourceContentType to link Source and ContentType models; removed contentType references from Source model; made changes to item controller and Source model to integrate new model --- .DS_Store | Bin 0 -> 10244 bytes .idea/inspectionProfiles/Project_Default.xml | 6 + .../inspectionProfiles/profiles_settings.xml | 0 .idea/misc.xml | 6 + .idea/modules.xml | 8 + .idea/sync-server.iml | 12 + .idea/vcs.xml | 6 + .idea/workspace.xml | 645 ++++++++ Gruntfile.js | 3 +- app/.DS_Store | Bin 0 -> 8196 bytes app/controllers/item.js | 1440 +++++++++-------- app/lib/.DS_Store | Bin 0 -> 6148 bytes app/lib/jsonapi.js | 25 +- app/lib/utils/debuggingTools | 52 + app/models/.DS_Store | Bin 0 -> 6148 bytes app/models/source.js | 101 +- app/models/sourceContentType.js | 28 + app/public/.DS_Store | Bin 0 -> 6148 bytes app/public/images/.DS_Store | Bin 0 -> 6148 bytes package.json | 4 +- 20 files changed, 1589 insertions(+), 747 deletions(-) create mode 100644 .DS_Store create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/sync-server.iml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml create mode 100644 app/.DS_Store create mode 100644 app/lib/.DS_Store create mode 100644 app/lib/utils/debuggingTools create mode 100644 app/models/.DS_Store create mode 100644 app/models/sourceContentType.js create mode 100644 app/public/.DS_Store create mode 100644 app/public/images/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..984dc05bf5abab75263180a636ecb887e3762937 GIT binary patch literal 10244 zcmeHNYitx%6h3EZamE2UMWocNcCjKpifnnb;scj<5qU{v3oVM2+1-(jPG^>#*%qkP z8buRD^cMzHNK8}=M*SfeA3qF6qcJg2QPCKqF&ZE75#N7IJonz&b!NA%CTc?BOfvV( z+;h%7_kQ=guh|6vY%gS60Ac_j(aoefiIPz+re}A>74dTBkVNtaaLp|(sf^v9tFR3- zLIgqtLIgqtLIgqtP7DNS&1RRR2pOg!0wDq+0wo04{1Bs?$%H3Igyd5PCHx3LvWl|# zMbC5&;65==COkPJBzL7U#pwY(SM-h;DBMXOW9B3io*WTUxC07zKrb?SClut-$uGvu z0TV)oX^23GKqUg~>|PBQKn8Tkm!IFK>`Z>o3-kH*+u3onxjz4w zfjD!nqPSbYD2qCM`?-FdFXo}mck-AeKVQTOT_9fFQ_PXo)qpIuR11Em*t0XJ^~W4u z$7A|@{ZR|Y%e8^Hz(JY-8ss3C9&~bP(@vAePwAX#IBqI1s9CcjE^bz$0N+a?)s zw=ONc-p~ho)oC5JlBazm)-8H>>5+<(vDr?-;sq z_@=$AUDS>uyfxyzlo|iJRh#}*JJULQawD~pHoS>E>Mq!Ip!$Abu_TIQP&n|64T4#!ko89vl%6RMLB*v zyn0mF1nMJpD-||n_)G!|Y(5e7LO*{hfXV9ZBj*2giWn0cKbNW}ZEd4I9oyC!=nzwF zn_jOR%c!xpeeLF zW1EG3i!`+v&CC?YSYKkADPxDQ&g5;=?6e)jHEe4`kE?HUnT{0{Q1x8x*m<{u7uI7? z`NC={i?JrhoSpT0-XST=ho;muo^{Tgd2N?0x%^m-T08ZWX^L_+&E1x3TxaO{b%SgX z*q}KX-O3u)mhJ(=G18{qWoiAo`l&1*n^AvSO|&6;Mp0AlA*oGLPa$n=Eg0_hV^h`I z8f9j*NmdU_8?)5F)pnKz%5qI~W^}eBUBmJ_@;ap=TRAVcx8SKNYm`4{2xC3v++wgY01MkK?cpvV?efSU_!pHGRd%zW`0s5sA_vCc{}szu ztX!2mVfp|56hZn^r`N@1oi}&EqQxC0Vw7>gPfN~3K{ZanBE@+32YF57BieAf(zuS8 zs&6YF9mP?jp2RpQvC$Nrd1hmy%E%~Q&z?=M5gISz_59{WMV&ySfe*cCyhUkLtBLqR zyRcPJ)G8VoLVMB0q@74(Kspw!KW&DRP}*f0wJPvH$=OsC(w^{d6M9@azXor?C-4pY z0KdWSSdAxR9meo%Y{phxj0x<(Bre4(u@l!~3cK)HqO!)RY@&rZPUj&U#+z{jZ^1ip z7w*P;@Lrwn8Ws1N<&}$2MvU&w zbhC@ujR?tIC}AcnpMQ?07xp=x@4+e0+jMh%!m}F@lDm^1cPA;n=KuFU2G9~d$^u<+ tZue~}ii>!KO<=JG-o6T(vQ5O8n~m`NKhBxP_t&!U{6E%t + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..e69de29 diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..28a804d --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..ee96d3c --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/sync-server.iml b/.idea/sync-server.iml new file mode 100644 index 0000000..24643cc --- /dev/null +++ b/.idea/sync-server.iml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..a83d7bd --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,645 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + kue + async.eachSe + queue + async.each + post + admin + route + geturl + perhaps + source + facebook + sources + populate + storeItemData + return + graph + itemDataObjectsFromPagePath + nodem + itemsGetUrl + series + sourceHos + properties + getitemurl + sourceCon + itemDataObjectsFromPagePath\: function\(contentType\) \{\n return templateCompiler\(this\.itemDataObjectsFromPagePathTemplate\, \{\n contentTypePluralCamelName\: contentType \? contentType\.pluralCamelName\(\) \: undefined\,\n contentTypePluralLowercaseName\: contentType \? contentType\.pluralLowercaseName\(\) \: undefined\n \}\)\;\n \}\,\n\n totalItemsAvailableFromPagePath\: function\(contentType\) \{\n return templateCompiler\(this\.totalItemsAvailableFromPagePathTemplate\, \{\n contentTypePluralCamelName\: contentType \? contentType\.pluralCamelName\(\) \: undefined\,\n contentTypePluralLowercaseName\: contentType \? contentType\.pluralLowercaseName\(\) \: undefined\n \}\)\;\n \}\, + attributes + validate + sourceContentType + storeAllForUserStorageSourceContentType + source.host + + + + + + + + + + + + + + + false + + false + false + true + + + true + DEFINITION_ORDER + + + + + + + + + + + + + + + Code quality toolsJavaScript + + + JavaScript + + + + + Eslint + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - 1501599967814 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 2227cd7e560327864d6ac26c00cdcc3e5816ce87 Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Thu, 17 Aug 2017 16:23:45 +0200 Subject: [PATCH 04/14] Correct ESLint errors from 'npm run test'; remove references to source.contentType from tests/controllers/item/itemsGetUrl.js in attempt to correct other test errors --- .gitignore | 4 +- Gruntfile.js | 1 - app/controllers/item.js | 1468 ++++++++++++------------- app/lib/jsonapi.js | 158 +-- app/models/item.js | 16 +- app/models/job.js | 20 +- app/models/source.js | 106 +- app/models/sourceContentType.js | 16 +- app/models/userSourceAuth.js | 4 +- tests/controllers/item/itemsGetUrl.js | 24 +- 10 files changed, 909 insertions(+), 908 deletions(-) diff --git a/.gitignore b/.gitignore index 88d5b21..9f7168c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ .cert* .env* data* -node_modules \ No newline at end of file +node_modules +.DS_Store +.idea/** diff --git a/Gruntfile.js b/Gruntfile.js index 9e46a21..f6d9021 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -44,7 +44,6 @@ module.exports = function(grunt) { nodemon: { dev: { script: 'app/server.js', - } }, symlink: { diff --git a/app/controllers/item.js b/app/controllers/item.js index 6d22fc8..b45fc3e 100644 --- a/app/controllers/item.js +++ b/app/controllers/item.js @@ -21,7 +21,6 @@ var async = require('async'); var debug = require('app/lib/debug')('syncServer:itemController'); var Item = require('app/models/item'); var Job = require('app/models/job'); -var SourceContentType = require('app/models/sourceContentType'); var kue = require('kue'); var logger = require('app/lib/logger'); var mime = require('app/lib/mime'); @@ -32,141 +31,139 @@ var UserSourceAuth = require('app/models/userSourceAuth'); var UserStorageAuth = require('app/models/userStorageAuth'); var validateParams = require('app/lib/validateParams'); -var util= require('util'); var queue = kue.createQueue(); -var tools = require('app/lib/utils/debuggingTools'); const STORE_ITEM_DATA = 'storeItemData'; -queue.process(STORE_ITEM_DATA, function(queueJob, done) { - console.log('process queueJob %s', queueJob.id); - - var getItem = (done) => { - Item.findById(queueJob.data.itemId, (error, item) => { - if (error) { - done(error); - } else if (!item) { - done(new Error('Item with queueJob.data.itemId not found')); - } else { - done(undefined, item); - } - }); - }; - - var getJob = (item, done) => { - Job.findById(queueJob.data.jobId, (error, job) => { - if (error) { - done(error); - } else if (!job) { - done(new Error('Job with queueJob.data.jobId not found')); - } else { - done(undefined, item, job); - } - }); - }; - - var storeItemData = (item, job, done) => { - debug.start('queueJob storeItemData', item.id, job.id); - module.exports.storeItemData(item, queueJob.data.data, job, done); - }; +queue.process(STORE_ITEM_DATA, function (queueJob, done) { + debug('process queueJob %s', queueJob.id); - async.waterfall([getItem, getJob, storeItemData], (error) => { - if (error) { - debug.error('failed to process queueJob %s: %s', queueJob.id, error.message); - } else { - debug.success('processed queueJob %s', queueJob.id); - } + var getItem = (done) => { + Item.findById(queueJob.data.itemId, (error, item) => { + if (error) { + done(error); + } else if (!item) { + done(new Error('Item with queueJob.data.itemId not found')); + } else { + done(undefined, item); + } + }); + }; + var getJob = (item, done) => { + Job.findById(queueJob.data.jobId, (error, job) => { + if (error) { done(error); + } else if (!job) { + done(new Error('Job with queueJob.data.jobId not found')); + } else { + done(undefined, item, job); + } }); + }; + + var storeItemData = (item, job, done) => { + debug.start('queueJob storeItemData', item.id, job.id); + module.exports.storeItemData(item, queueJob.data.data, job, done); + }; + + async.waterfall([getItem, getJob, storeItemData], (error) => { + if (error) { + debug.error('failed to process queueJob %s: %s', queueJob.id, error.message); + } else { + debug.success('processed queueJob %s', queueJob.id); + } + + done(error); + }); }); queue.on('error', (error) => { - debug.error('queueJob failed:', error.message); + debug.error('queueJob failed:', error.message); }); /** * Callback resource found at URL. - * (Get resource from passed URL and pass to callback (done) + *That is: get resource from passed URL and pass to callback (done) * @param {string} url - URL of resource with extension that corresponds to a supported media type. * @param {module:controllers/item~resourceCallback} done */ -module.exports.getResource = function(url, done) { - var log = logger.scopedLog(); - - var validate = function(done) { - validateParams([{ - name: 'url', variable: url, required: true, requiredType: 'string', regex: urlRegex - }], function(error) { - if (!error && module.exports.hasSupportedMediaType(url) === false) { - error = new Error('Parameter url indicates unsupported media type'); - } - - done(error); - }); - }; +module.exports.getResource = function (url, done) { + var log = logger.scopedLog(); - var setupLog = function(done) { - debug.start('getResource %s', url); + var validate = function (done) { + validateParams([{ + name: 'url', variable: url, required: true, requiredType: 'string', regex: urlRegex + }], function (error) { + if (!error && module.exports.hasSupportedMediaType(url) === false) { + error = new Error('Parameter url indicates unsupported media type'); + } - log = logger.scopedLog({ - url: url - }); + done(error); + }); + }; - done(); - }; + var setupLog = function (done) { + debug.start('getResource %s', url); - var getResource = function(done) { - var mediaType = mime.lookup(url); - mediaType = mediaType ? mediaType : 'application/json'; - - request({ - url: url, - headers: { - 'Content-Type': mediaType - } - }, function(error, res, body) { - if (error) { - return done(error); - } else if (request.statusCodeError(res.statusCode)) { - return done(request.statusCodeError(res.statusCode)); - } - - var resource; - - switch(mediaType) { - case 'image/jpeg': - resource = new Buffer(body); - break; - case 'application/json': - try { - resource = JSON.parse(body); - } catch (error) { - return done(new Error('Unable to parse resource')); - } - break; - default: - return done('Unrecognized media type encountered'); - } - - debug.success('getResource (mediaType: %s)', mediaType); - done(undefined, resource); - }); - }; + log = logger.scopedLog({ + url: url + }); - async.waterfall([ - validate, - setupLog, - getResource - ], function(error, resource) { - if (error) { - log('error', 'Item controller failed to get resource', { error: error }); + done(); + }; + + var getResource = function (done) { + var mediaType = mime.lookup(url); + mediaType = mediaType ? mediaType : 'application/json'; + + request({ + url: url, + headers: { + 'Content-Type': mediaType + } + }, function (error, res, body) { + if (error) { + return done(error); + } else if (request.statusCodeError(res.statusCode)) { + return done(request.statusCodeError(res.statusCode)); + } + + var resource; + + switch (mediaType) { + case 'image/jpeg': + resource = new Buffer(body); + break; + case 'application/json': + try { + resource = JSON.parse(body); + } catch (error) { + return done(new Error('Unable to parse resource')); } + break; + default: + return done('Unrecognized media type encountered'); + } - done(error, resource); + debug.success('getResource (mediaType: %s)', mediaType); + done(undefined, resource); }); + }; + + async.waterfall([ + validate, + setupLog, + getResource + ], function (error, resource) { + if (error) { + log('error', 'Item controller failed to get resource', {error: error}); + } + + done(error, resource); + }); }; /** @@ -174,21 +171,21 @@ module.exports.getResource = function(url, done) { * @param {string} url - URL * @returns {boolean|undefined} Whether media type supported by controller operations */ -module.exports.hasSupportedMediaType = function(url) { - validateParams([{ - name: 'url', variable: url, required: true, requiredType: 'string' - }]); +module.exports.hasSupportedMediaType = function (url) { + validateParams([{ + name: 'url', variable: url, required: true, requiredType: 'string' + }]); - var pathname = Url.parse(url).pathname; - var lastSegment = (pathname.lastIndexOf('/') !== -1) ? pathname.substr(pathname.lastIndexOf('/') + 1) : pathname; + var pathname = Url.parse(url).pathname; + var lastSegment = (pathname.lastIndexOf('/') !== -1) ? pathname.substr(pathname.lastIndexOf('/') + 1) : pathname; - if (lastSegment.indexOf('.') === -1) { - return; - } + if (lastSegment.indexOf('.') === -1) { + return; + } - debug('hasSupportedMediaType url %s, mime %s', lastSegment, mime.lookup(lastSegment)); + debug('hasSupportedMediaType url %s, mime %s', lastSegment, mime.lookup(lastSegment)); - return (['image/jpeg', 'application/json'].indexOf(mime.lookup(lastSegment)) !== -1); + return (['image/jpeg', 'application/json'].indexOf(mime.lookup(lastSegment)) !== -1); }; /** @@ -198,22 +195,22 @@ module.exports.hasSupportedMediaType = function(url) { * @param {ContentType} contentType * @returns {Object[]} ItemDataObjects */ -module.exports.itemDataObjectsFromPage = function(page, source, contentType) { - validateParams([{ - name: 'page', variable: page, required: true, requiredType: 'object' - }, { - name: 'source', variable: source, required: true, requiredProperties: ['itemDataObjectsFromPagePathTemplate'] - }]); +module.exports.itemDataObjectsFromPage = function (page, source, contentType) { + validateParams([{ + name: 'page', variable: page, required: true, requiredType: 'object' + }, { + name: 'source', variable: source, required: true, requiredProperties: ['itemDataObjectsFromPagePathTemplate'] + }]); - var path = source.itemDataObjectsFromPagePath(contentType); + var path = source.itemDataObjectsFromPagePath(contentType); - debug.trace('itemDataObjectsFromPage path: %s', path); + debug.trace('itemDataObjectsFromPage path: %s', path); - var itemDataObjects = path ? _.get(page, path, []) : page; + var itemDataObjects = path ? _.get(page, path, []) : page; - debug.trace('itemDataObjectsFromPage total: %s', itemDataObjects.length); + debug.trace('itemDataObjectsFromPage total: %s', itemDataObjects.length); - return itemDataObjects; + return itemDataObjects; }; /** @@ -224,33 +221,36 @@ module.exports.itemDataObjectsFromPage = function(page, source, contentType) { * @param {Object} pagination - Pagination used to make request. * @returns {string} URL for making a GET request */ -module.exports.itemsGetUrl = function(source, sourceContentType, userSourceAuth, pagination) { - validateParams([{ - name: 'source', variable: source, required: true, requiredProperties: ['host'] - }, { - name: 'sourceContentType', variable: sourceContentType, required: true, requiredProperties: ['contentType','itemsGetUrlTemplate'] - }, { - name: 'userSourceAuth', variable: userSourceAuth, required: true, requiredProperties: ['sourceToken'] - }, { - name: 'pagination', variable: pagination - }]); - - - debug("getItemURL, source.authScope = %s",source.authScope); - - return source.itemsGetUrl( sourceContentType.itemsGetUrlTemplate, { - sourceToken: userSourceAuth.sourceToken, - apiVersion: source.apiVersion, - contentTypePluralCamelName: sourceContentType.contentType.pluralCamelName(), - contentTypePluralLowercaseName: sourceContentType.contentType.pluralLowercaseName(), - sourceHost: source.host, - sourceItemsLimit: source.itemsLimit, - maxId: (typeof pagination !== 'undefined' && pagination.maxId) ? pagination.maxId : undefined, - offset: (typeof pagination !== 'undefined' && pagination.offset) ? pagination.offset : 0, - next: (typeof pagination !== 'undefined' && pagination.next) ? pagination.next : undefined, - sourceName: source.name - - }); +module.exports.itemsGetUrl = function (source, sourceContentType, userSourceAuth, pagination) { + validateParams([{ + name: 'source', variable: source, required: true, requiredProperties: ['host'] + }, { + name: 'sourceContentType', + variable: sourceContentType, + required: true, + requiredProperties: ['contentType', 'itemsGetUrlTemplate'] + }, { + name: 'userSourceAuth', variable: userSourceAuth, required: true, requiredProperties: ['sourceToken'] + }, { + name: 'pagination', variable: pagination + }]); + + + debug('getItemURL, source.authScope = %s', source.authScope); + + return source.itemsGetUrl(sourceContentType.itemsGetUrlTemplate, { + sourceToken: userSourceAuth.sourceToken, + apiVersion: source.apiVersion, + contentTypePluralCamelName: sourceContentType.contentType.pluralCamelName(), + contentTypePluralLowercaseName: sourceContentType.contentType.pluralLowercaseName(), + sourceHost: source.host, + sourceItemsLimit: source.itemsLimit, + maxId: (typeof pagination !== 'undefined' && pagination.maxId) ? pagination.maxId : undefined, + offset: (typeof pagination !== 'undefined' && pagination.offset) ? pagination.offset : 0, + next: (typeof pagination !== 'undefined' && pagination.next) ? pagination.next : undefined, + sourceName: source.name + + }); }; /** @@ -261,49 +261,48 @@ module.exports.itemsGetUrl = function(source, sourceContentType, userSourceAuth, * @param {Object} contentType - ContentType of items. * @returns {number} Total number of items available. */ -module.exports.totalItemsAvailableFromPage = function(page, source, contentType) { - validateParams([{ - name: 'page', variable: page, required: true, requiredType: 'object' - }, { - name: 'source', variable: source, required: true - }]); +module.exports.totalItemsAvailableFromPage = function (page, source, contentType) { + validateParams([{ + name: 'page', variable: page, required: true, requiredType: 'object' + }, { + name: 'source', variable: source, required: true + }]); - var path = source.totalItemsAvailableFromPagePath(contentType); + var path = source.totalItemsAvailableFromPagePath(contentType); - debug.trace('totalItemsAvailableFromPage path: %s', path); + debug.trace('totalItemsAvailableFromPage path: %s', path); - var total = path ? _.get(page, path) : page; + var total = path ? _.get(page, path) : page; - debug.trace('totalItemsAvailableFromPage total: %s', total); + debug.trace('totalItemsAvailableFromPage total: %s', total); - return total; + return total; }; - /** * Returns error from items page if error exists within. * @param {Object} page - Items page. * @returns {error} Error */ -module.exports.itemsPageError = function(page) { - validateParams([{ - name: 'page', variable: page, required: true, requiredType: 'object' - }]); - - if (page.meta && page.meta.code && Number(page.meta.code) >= 400) { - var message; - - if (page.meta.errorDetail) { - message = `${page.meta.errorDetail} (${page.meta.code})`; - } else if (page.meta.errorType) { - message = `HTTP status code ${page.meta.code}, ${page.meta.errorType}`; - } else { - message = `HTTP status code ${page.meta.code}`; - } - - return new Error(message); +module.exports.itemsPageError = function (page) { + validateParams([{ + name: 'page', variable: page, required: true, requiredType: 'object' + }]); + + if (page.meta && page.meta.code && Number(page.meta.code) >= 400) { + var message; + + if (page.meta.errorDetail) { + message = `${page.meta.errorDetail} (${page.meta.code})`; + } else if (page.meta.errorType) { + message = `HTTP status code ${page.meta.code}, ${page.meta.errorType}`; + } else { + message = `HTTP status code ${page.meta.code}`; } + + return new Error(message); + } }; @@ -314,40 +313,40 @@ module.exports.itemsPageError = function(page) { * @param {Object} contentType - ContentType of current items page. * @returns {Object} Pagination for next items page. */ -module.exports.itemsPageNextPagination = function(page, pagination, contentType) { - validateParams([{ - name: 'page', variable: page, required: true, requiredType: 'object' - }, { - name: 'contentType', variable: contentType, requiredProperties: ['pluralCamelName'] - }]); - - var nextPagination; - - debug.start('itemsPageNextPagination (pagination: %o)', pagination); - - if (page.response && page.response[contentType.pluralLowercaseName()] && page.response[contentType.pluralLowercaseName()].items && page.response[contentType.pluralLowercaseName()].items.length) { - if (pagination && pagination.offset) { - nextPagination = { offset: pagination.offset + page.response[contentType.pluralLowercaseName()].items.length }; - } else { - nextPagination = { offset: page.response[contentType.pluralLowercaseName()].items.length }; - } +module.exports.itemsPageNextPagination = function (page, pagination, contentType) { + validateParams([{ + name: 'page', variable: page, required: true, requiredType: 'object' + }, { + name: 'contentType', variable: contentType, requiredProperties: ['pluralCamelName'] + }]); + + var nextPagination; + + debug.start('itemsPageNextPagination (pagination: %o)', pagination); + + if (page.response && page.response[contentType.pluralLowercaseName()] && page.response[contentType.pluralLowercaseName()].items && page.response[contentType.pluralLowercaseName()].items.length) { + if (pagination && pagination.offset) { + nextPagination = {offset: pagination.offset + page.response[contentType.pluralLowercaseName()].items.length}; + } else { + nextPagination = {offset: page.response[contentType.pluralLowercaseName()].items.length}; } + } - if (page.data && page.data.pagination && page.data.pagination.next_max_id) { - nextPagination = { maxId: page.data.pagination.next_max_id }; - } + if (page.data && page.data.pagination && page.data.pagination.next_max_id) { + nextPagination = {maxId: page.data.pagination.next_max_id}; + } - if (page.links && page.links.next) { - nextPagination = { next: page.links.next }; - } + if (page.links && page.links.next) { + nextPagination = {next: page.links.next}; + } - if (page.paging && page.paging.next) { - nextPagination = { next: page.paging.next }; - } + if (page.paging && page.paging.next) { + nextPagination = {next: page.paging.next}; + } - debug.success('itemsPageNextPagination (nextPagination: %o)', nextPagination); + debug.success('itemsPageNextPagination (nextPagination: %o)', nextPagination); - return nextPagination; + return nextPagination; }; /** @@ -356,20 +355,20 @@ module.exports.itemsPageNextPagination = function(page, pagination, contentType) * @param {Object} data - Raw item data from source. * @param {function} done - Error-first callback function expecting file system path as second parameter. */ -module.exports.storagePath = function(item, data, done) { - var validate = function(done) { - validateParams([{ - name: 'item', variable: item, required: true, requiredProperties: ['id', 'contentType'] - }], done); - }; +module.exports.storagePath = function (item, data, done) { + var validate = function (done) { + validateParams([{ + name: 'item', variable: item, required: true, requiredProperties: ['id', 'contentType'] + }], done); + }; - var storagePath = function(done) { - var path = '/' + item.source.kebabName() + '/' + item.contentType.pluralKebabName() + '/' + item.slug(data) + '.json'; - debug.success('storagePath: %s', path); - done(undefined, path); - }; + var storagePath = function (done) { + var path = '/' + item.source.kebabName() + '/' + item.contentType.pluralKebabName() + '/' + item.slug(data) + '.json'; + debug.success('storagePath: %s', path); + done(undefined, path); + }; - async.waterfall([validate, storagePath], done); + async.waterfall([validate, storagePath], done); }; /** @@ -382,64 +381,62 @@ module.exports.storagePath = function(item, data, done) { * @param {Job} [job] - Job for which to store items. * @param {callback} done */ -module.exports.storeAllForUserStorageSource = function(user, source, storage, job, done) { - var log = logger.scopedLog(); - - var validate = function(done) { - validateParams([{ - name: 'user', variable: user, required: true, requiredProperties: ['id'] - }, { - name: 'source', variable: source, required: true, requiredProperties: ['id'] - }, { - name: 'storage', variable: storage, required: true, requiredProperties: ['id'] - }], done); - }; - - var setupLog = function(done) { - debug.start('storeAllForUserStorageSource'); +module.exports.storeAllForUserStorageSource = function (user, source, storage, job, done) { + var log = logger.scopedLog(); - log = logger.scopedLog({ - user: user.id, - source: source.id, - storage: storage.id - }); + var validate = function (done) { + validateParams([{ + name: 'user', variable: user, required: true, requiredProperties: ['id'] + }, { + name: 'source', variable: source, required: true, requiredProperties: ['id'] + }, { + name: 'storage', variable: storage, required: true, requiredProperties: ['id'] + }], done); + }; - done(); - }; + var setupLog = function (done) { + debug.start('storeAllForUserStorageSource'); - var storeAllForUserStorageSourceContentType = function(sourceContentType, done) { - module.exports.storeAllForUserStorageSourceContentType(user, source, storage, sourceContentType, job, done); - }; + log = logger.scopedLog({ + user: user.id, + source: source.id, + storage: storage.id + }); + done(); + }; + var storeAllForUserStorageSourceContentType = function (sourceContentType, done) { + module.exports.storeAllForUserStorageSourceContentType(user, source, storage, sourceContentType, job, done); + }; - let getSourceContentTypesFunction = function(done) { - source.getSourceContentTypesForSource(function(err,sourceContentTypes) { - if (err) { - return done(err); - } else { - done(err,sourceContentTypes); - } - }); - }; + let getSourceContentTypesFunction = function (done) { - var storeAllItems = function(sourceContentTypes,done) { - debug.start('storeAllItems (sourceContentTypes: %s)', sourceContentTypes.length); - async.eachSeries(sourceContentTypes, storeAllForUserStorageSourceContentType, done); - }; + source.getSourceContentTypesForSource(function (err, sourceContentTypes) { + if (err) { + return done(err); + } else { + done(err, sourceContentTypes); + } + }); + }; + var storeAllItems = function (sourceContentTypes, done) { + debug.start('storeAllItems (sourceContentTypes: %s)', sourceContentTypes.length); + async.eachSeries(sourceContentTypes, storeAllForUserStorageSourceContentType, done); + }; - async.waterfall([validate, setupLog,getSourceContentTypesFunction, storeAllItems], function(error) { - if (error) { - log('error', 'Item controller failed to store all items', { error: error.message }); - } else { - debug.success('storeAllForUserStorageSource'); - } + async.waterfall([validate, setupLog, getSourceContentTypesFunction, storeAllItems], function (error) { + if (error) { + log('error', 'Item controller failed to store all items', {error: error.message}); + } else { + debug.success('storeAllForUserStorageSource'); + } - done(error); - }); + done(error); + }); }; /** @@ -453,64 +450,64 @@ module.exports.storeAllForUserStorageSource = function(user, source, storage, jo * @param {Job} [job] - Job for which to store items. * @param {callback} done */ -module.exports.storeAllForUserStorageSourceContentType = function(user, source, storage, sourceContentType, job, done) { - debug('storeAllForUserStorageSourceContentType, source = %s, storage = %s, sourceContentType = %s', source, storage, sourceContentType); - var log = logger.scopedLog(); - - var validate = function(done) { - validateParams([{ - name: 'user', variable: user, required: true, requiredProperties: ['id'] - }, { - name: 'source', variable: source, required: true, requiredProperties: ['id'] - }, { - name: 'storage', variable: storage, required: true, requiredProperties: ['id'] - }, { - name: 'sourceContentType', variable: sourceContentType, required: true, requiredProperties: ['id'] - }], done); - }; +module.exports.storeAllForUserStorageSourceContentType = function (user, source, storage, sourceContentType, job, done) { + debug('storeAllForUserStorageSourceContentType, source = %s, storage = %s, sourceContentType = %s', source, storage, sourceContentType); + var log = logger.scopedLog(); - var setupLog = function(done) { - debug.start('storeAllForUserStorageSourceContentType'); + var validate = function (done) { + validateParams([{ + name: 'user', variable: user, required: true, requiredProperties: ['id'] + }, { + name: 'source', variable: source, required: true, requiredProperties: ['id'] + }, { + name: 'storage', variable: storage, required: true, requiredProperties: ['id'] + }, { + name: 'sourceContentType', variable: sourceContentType, required: true, requiredProperties: ['id'] + }], done); + }; + + var setupLog = function (done) { + debug.start('storeAllForUserStorageSourceContentType'); + + log = logger.scopedLog({ + user: user.id, + source: source.id, + storage: storage.id, + sourceContentType: sourceContentType.id + }); - log = logger.scopedLog({ - user: user.id, - source: source.id, - storage: storage.id, - sourceContentType: sourceContentType.id - }); + done(); + }; - done(); + var storeAllItems = function (done) { + var storeAllItemPages = function myself(error, pagination) { + if (error) { + if (done) { + done(error); + } + } else { + if (pagination) { + module.exports.storeItemsPage(user, source, storage, sourceContentType, pagination, job, myself); + } else if (done) { + done(); + } + } }; - var storeAllItems = function(done) { - var storeAllItemPages = function myself(error, pagination) { - if (error) { - if (done) { - done(error); - } - } else { - if (pagination) { - module.exports.storeItemsPage(user, source, storage, sourceContentType, pagination, job, myself); - } else if (done) { - done(); - } - } - }; - - storeAllItemPages(null, { offset: 0 }); - }; + storeAllItemPages(null, {offset: 0}); + }; - async.series([validate, setupLog, storeAllItems], function(error) { - if (error) { - debug.error('storeAllForUserStorageSourceContentType (message: %s)', error.message); - log('error', 'Item controller failed to store all items', { error: error }); - } else { - debug.success('storeAllForUserStorageSourceContentType'); - log('milestone', 'Item controller stored all items', { error: error }); - } + async.series([validate, setupLog, storeAllItems], function (error) { + if (error) { + debug.error('storeAllForUserStorageSourceContentType (message: %s)', error.message); + log('error', 'Item controller failed to store all items', {error: error}); + } else { + debug.success('storeAllForUserStorageSourceContentType'); + log('milestone', 'Item controller stored all items', {error: error}); + } - done(error); - }); + done(error); + }); }; /** @@ -525,149 +522,149 @@ module.exports.storeAllForUserStorageSourceContentType = function(user, source, * @param {Job} [job] - Job for which to store items. * @param {callback} done */ -module.exports.storeItemsPage = function(user, source, storage, sourceContentType, pagination, job, done) { - - var log = logger.scopedLog(); - var ids, page, userSourceAuth; - - var validate = function(done) { - validateParams([{ - name: 'user', variable: user, required: true, requiredProperties: ['id'] - }, { - name: 'source', variable: source, required: true, requiredProperties: ['id'] - }, { - name: 'storage', variable: storage, required: true, requiredProperties: ['id'] - }, { - name: 'sourceContentType', variable: sourceContentType, required: true, requiredProperties: ['id'] - }, { - name: 'pagination', variable: pagination, required: true - }], done); - }; - - var setupLog = function(done) { - debug.start('## storeItemsPage (sourceContentType: %s, pagination: %o)', sourceContentType.id, pagination); +module.exports.storeItemsPage = function (user, source, storage, sourceContentType, pagination, job, done) { - ids = { - user: user.id, - storage: storage.id, - source: source.id, - sourceContentType: sourceContentType.id - }; + var log = logger.scopedLog(); + var ids, page, userSourceAuth; - log = logger.scopedLog(Object.assign({}, pagination, ids)); - done(); - }; - - var findUserSourceAuth = function(done) { - UserSourceAuth.findOne({ - user: user.id, - source: source.id - }, function(error, foundUserSourceAuth) { - if (!foundUserSourceAuth && !error) { - error = new Error('Failed to find userSourceAuth'); - } - - userSourceAuth = foundUserSourceAuth; - done(error); - }); - }; - - // get the page of date from URL - // whcih will be converted INTO items - - var getItemsPageResource = function(done) { - module.exports.getResource(module.exports.itemsGetUrl(source, sourceContentType, userSourceAuth, pagination), done); - }; - - var getItemDataObjects = function(resource, done) { - page = resource; - var error = module.exports.itemsPageError(page); + var validate = function (done) { + validateParams([{ + name: 'user', variable: user, required: true, requiredProperties: ['id'] + }, { + name: 'source', variable: source, required: true, requiredProperties: ['id'] + }, { + name: 'storage', variable: storage, required: true, requiredProperties: ['id'] + }, { + name: 'sourceContentType', variable: sourceContentType, required: true, requiredProperties: ['id'] + }, { + name: 'pagination', variable: pagination, required: true + }], done); + }; + + var setupLog = function (done) { + debug.start('## storeItemsPage (sourceContentType: %s, pagination: %o)', sourceContentType.id, pagination); + + ids = { + user: user.id, + storage: storage.id, + source: source.id, + sourceContentType: sourceContentType.id + }; + + log = logger.scopedLog(Object.assign({}, pagination, ids)); + done(); + }; + + var findUserSourceAuth = function (done) { + UserSourceAuth.findOne({ + user: user.id, + source: source.id + }, function (error, foundUserSourceAuth) { + if (!foundUserSourceAuth && !error) { + error = new Error('Failed to find userSourceAuth'); + } + + userSourceAuth = foundUserSourceAuth; + done(error); + }); + }; - if (error) { - return done(new Error('Failed to retrieve valid item objects page. ' + error.message)); - } + // get the page of date from URL + // whcih will be converted INTO items - var itemDataObjects = module.exports.itemDataObjectsFromPage(page, source, sourceContentType.contentType); - var totalItemsAvailable = module.exports.totalItemsAvailableFromPage(page, source, sourceContentType.contentType); + var getItemsPageResource = function (done) { + module.exports.getResource(module.exports.itemsGetUrl(source, sourceContentType, userSourceAuth, pagination), done); + }; - if (job && totalItemsAvailable && pagination.offset === 0) { - job.updateTotalItemsAvailable(totalItemsAvailable); - } + var getItemDataObjects = function (resource, done) { + page = resource; + var error = module.exports.itemsPageError(page); - if (!itemDataObjects || !itemDataObjects.length) { - debug.warning('storeItemsPage retrieved page with no data (sourceContentType: %s, pagination: %o)', sourceContentType.id, pagination); - } + if (error) { + return done(new Error('Failed to retrieve valid item objects page. ' + error.message)); + } - done(undefined, itemDataObjects); - }; + var itemDataObjects = module.exports.itemDataObjectsFromPage(page, source, sourceContentType.contentType); + var totalItemsAvailable = module.exports.totalItemsAvailableFromPage(page, source, sourceContentType.contentType); - var persistItemDataObjects = function(itemDataObjects, done) { - var count = 0; - async.mapSeries(itemDataObjects, function(itemDataObject, done) { - count++; - debug('persistItemDataObject #%s', count); - - // this creates the Item mongo data document - module.exports.persistItemDataObject(itemDataObject, { - user: user, - storage: storage, - source: source, - sourceContentType: sourceContentType - }, (error, item) => { - done(error, { - item: item, - data: itemDataObject - }); - }); - }, done); - }; + if (job && totalItemsAvailable && pagination.offset === 0) { + job.updateTotalItemsAvailable(totalItemsAvailable); + } - var storeItemsData = function(itemPairs, done) { - async.each(itemPairs, function(itemPair, done) { - var jobAttributes = { - itemId: itemPair.item.id, - data: itemPair.data - }; + if (!itemDataObjects || !itemDataObjects.length) { + debug.warning('storeItemsPage retrieved page with no data (sourceContentType: %s, pagination: %o)', sourceContentType.id, pagination); + } - if (job) { - jobAttributes.jobId = job.id; - } + done(undefined, itemDataObjects); + }; + + var persistItemDataObjects = function (itemDataObjects, done) { + var count = 0; + async.mapSeries(itemDataObjects, function (itemDataObject, done) { + count++; + debug('persistItemDataObject #%s', count); + + // this creates the Item mongo data document + module.exports.persistItemDataObject(itemDataObject, { + user: user, + storage: storage, + source: source, + sourceContentType: sourceContentType + }, (error, item) => { + done(error, { + item: item, + data: itemDataObject + }); + }); + }, done); + }; + + var storeItemsData = function (itemPairs, done) { + async.each(itemPairs, function (itemPair, done) { + var jobAttributes = { + itemId: itemPair.item.id, + data: itemPair.data + }; + + if (job) { + jobAttributes.jobId = job.id; + } // this is where the actual jobs queue "items" are created - /// where the magic happens… - var queueJob = queue.create(STORE_ITEM_DATA, jobAttributes).save((error) => { - if (error) { - debug.error('queueJob %s failed to queue: %s', queueJob.id, error.message); - } else { - debug.success('queueJob %s queued for item %s', queueJob.id, itemPair.item.id); - } - }); - - done(); - }, done); - }; - - var determineNextPagination = function(done) { - done(undefined, module.exports.itemsPageNextPagination(page, pagination, sourceContentType.contentType)); - }; - - async.waterfall([ - validate, - setupLog, - findUserSourceAuth, - getItemsPageResource, - getItemDataObjects, - persistItemDataObjects, - storeItemsData, - determineNextPagination - ], function(error, nextPagination) { + /// where the magic happens… + var queueJob = queue.create(STORE_ITEM_DATA, jobAttributes).save((error) => { if (error) { - log('error', 'Item controller failed to store page of items', { error: error.message }); + debug.error('queueJob %s failed to queue: %s', queueJob.id, error.message); } else { - debug.success('storeItemsPage (sourceContentType: %s, pagination: %o, nextPagination: %o)', sourceContentType.id, pagination, nextPagination); + debug.success('queueJob %s queued for item %s', queueJob.id, itemPair.item.id); } + }); + + done(); + }, done); + }; + + var determineNextPagination = function (done) { + done(undefined, module.exports.itemsPageNextPagination(page, pagination, sourceContentType.contentType)); + }; + + async.waterfall([ + validate, + setupLog, + findUserSourceAuth, + getItemsPageResource, + getItemDataObjects, + persistItemDataObjects, + storeItemsData, + determineNextPagination + ], function (error, nextPagination) { + if (error) { + log('error', 'Item controller failed to store page of items', {error: error.message}); + } else { + debug.success('storeItemsPage (sourceContentType: %s, pagination: %o, nextPagination: %o)', sourceContentType.id, pagination, nextPagination); + } - done(error, nextPagination); - }); + done(error, nextPagination); + }); }; /** @@ -680,120 +677,123 @@ module.exports.storeItemsPage = function(user, source, storage, sourceContentTyp */ // this creates the Item _about_ the data we're about to store, and saves this MongoDBx -module.exports.persistItemDataObject = function(itemDataObject, relationships, done) { - var conditions; - var log = logger.scopedLog(); - - var validate = function(done) { - validateParams([{ - name: 'itemDataObject', variable: itemDataObject, required: true, requiredProperties: ['id'] - }, { - name: 'relationships', variable: relationships, required: true, requiredProperties: ['user', 'storage', 'source', 'sourceContentType'] - }], done); - }; +module.exports.persistItemDataObject = function (itemDataObject, relationships, done) { + var conditions; + var log = logger.scopedLog(); - var compileConditions = function(done) { - debug.start('persistItemDataObject'); - - conditions = { - user: relationships.user.id, - storage: relationships.storage.id, - source: relationships.source.id, - contentType: relationships.sourceContentType.contentType.id, - sourceItem: itemDataObject.id, - }; - done(); - }; + var validate = function (done) { + validateParams([{ + name: 'itemDataObject', variable: itemDataObject, required: true, requiredProperties: ['id'] + }, { + name: 'relationships', + variable: relationships, + required: true, + requiredProperties: ['user', 'storage', 'source', 'sourceContentType'] + }], done); + }; + + var compileConditions = function (done) { + debug.start('persistItemDataObject'); + + conditions = { + user: relationships.user.id, + storage: relationships.storage.id, + source: relationships.source.id, + contentType: relationships.sourceContentType.contentType.id, + sourceItem: itemDataObject.id, + }; + done(); + }; + + var setupLog = function (done) { + log = logger.scopedLog(conditions); + done(); + }; + + var persistItemDataObject = function (done) { + Item.findOrCreate(conditions, function (error, item) { + if (error) { + done(error); + } else { + done(undefined, item); + } + }); + }; - var setupLog = function(done) { - log = logger.scopedLog(conditions); - done(); - }; + var saveSourceCreatedAt = function (item, done) { + var createdAt = itemDataObject.createdAt ? itemDataObject.createdAt * 1000 : null; + createdAt = !createdAt && itemDataObject.created_time ? itemDataObject.created_time : createdAt; - var persistItemDataObject = function(done) { - Item.findOrCreate(conditions, function(error, item) { - if (error) { - done(error); - } else { - done(undefined, item); - } - }); - }; + if (createdAt) { + item.sourceCreatedAt = new Date(createdAt); + item.save((error) => { + done(error, item); + }); + } else { + done(undefined, item); + } + }; - var saveSourceCreatedAt = function(item, done) { - var createdAt = itemDataObject.createdAt ? itemDataObject.createdAt * 1000 : null; - createdAt = !createdAt && itemDataObject.created_time ? itemDataObject.created_time : createdAt; + var saveDescription = function (item, done) { + if (itemDataObject) { + var parts = []; - if (createdAt) { - item.sourceCreatedAt = new Date(createdAt); - item.save((error) => { - done(error, item); - }); - } else { - done(undefined, item); + if (itemDataObject.venue && itemDataObject.venue.name) { + parts.push(itemDataObject.venue.name); + } else if (itemDataObject.firstName || itemDataObject.lastName) { + if (itemDataObject.firstName) { + parts.push(itemDataObject.firstName); } - }; - var saveDescription = function(item, done) { - if (itemDataObject) { - var parts = []; - - if (itemDataObject.venue && itemDataObject.venue.name) { - parts.push(itemDataObject.venue.name); - } else if (itemDataObject.firstName || itemDataObject.lastName) { - if (itemDataObject.firstName) { - parts.push(itemDataObject.firstName); - } - - if (itemDataObject.lastName) { - parts.push(itemDataObject.lastName); - } - } else if (itemDataObject.text) { - parts.push(itemDataObject.text); - } else if (itemDataObject.message) { - parts.push(itemDataObject.message); - } - - item.description = parts.join(' '); - item.save((error) => { - done(error, item); - }); - } else { - done(undefined, item); + if (itemDataObject.lastName) { + parts.push(itemDataObject.lastName); } - }; - - var determinePath = function(item, done) { - module.exports.storagePath(item, itemDataObject, function(error, path) { - done(error, path, item); - }); - }; - - var savePath = function(path, item, done) { - item.storagePath = path; - item.save((error) => { - done(error, item); - }); - }; + } else if (itemDataObject.text) { + parts.push(itemDataObject.text); + } else if (itemDataObject.message) { + parts.push(itemDataObject.message); + } + + item.description = parts.join(' '); + item.save((error) => { + done(error, item); + }); + } else { + done(undefined, item); + } + }; - async.waterfall([ - validate, - compileConditions, - setupLog, - persistItemDataObject, - saveSourceCreatedAt, - saveDescription, - determinePath, - savePath - ], function(error, item) { - if (error) { - log('error', 'Item controller failed to persist item data object', { error: error }); - } else { - debug.success('persistItemDataObject', item); - } + var determinePath = function (item, done) { + module.exports.storagePath(item, itemDataObject, function (error, path) { + done(error, path, item); + }); + }; - done(error, item); + var savePath = function (path, item, done) { + item.storagePath = path; + item.save((error) => { + done(error, item); }); + }; + + async.waterfall([ + validate, + compileConditions, + setupLog, + persistItemDataObject, + saveSourceCreatedAt, + saveDescription, + determinePath, + savePath + ], function (error, item) { + if (error) { + log('error', 'Item controller failed to persist item data object', {error: error}); + } else { + debug.success('persistItemDataObject', item); + } + + done(error, item); + }); }; /** @@ -807,108 +807,108 @@ module.exports.persistItemDataObject = function(itemDataObject, relationships, d * @param {callback} done */ // item is newly created item, data is the data for that item (from source) -module.exports.storeItemData = function(item, data, job, done) { - var log = logger.scopedLog(); - - var validate = function(done) { - validateParams([{ - name: 'item', variable: item, required: true, requiredProperties: ['user', 'storage', 'save'] - }, { - name: 'data', variable: data, required: true, requiredType: 'object' - }], done); - }; - - var setupLog = function(done) { - debug.start('storeItemData'); - log = logger.scopedLog({ item: item.id }); - done(); - }; +module.exports.storeItemData = function (item, data, job, done) { + var log = logger.scopedLog(); - var updateStorageAttemptedAt = function(done) { - item.storageAttemptedAt = Date.now(); - item.save(function(error) { - done(error); - }); - }; + var validate = function (done) { + validateParams([{ + name: 'item', variable: item, required: true, requiredProperties: ['user', 'storage', 'save'] + }, { + name: 'data', variable: data, required: true, requiredType: 'object' + }], done); + }; + + var setupLog = function (done) { + debug.start('storeItemData'); + log = logger.scopedLog({item: item.id}); + done(); + }; + + var updateStorageAttemptedAt = function (done) { + item.storageAttemptedAt = Date.now(); + item.save(function (error) { + done(error); + }); + }; // ?? what is going on here?? - var storeFile = function(done) { - debug("storeFile : item.user = ",item.user); - module.exports.storeFile(item.user, item.storage, item.storagePath, data, (error, storeFileResult) => { - if (error) { - debug.error('storeFile item %s, error %o, storeFileResult %o', item.id, error, storeFileResult); - item.storageError = error.message; - item.storageFailedAt = Date.now(); - item.save(() => { - done(error, storeFileResult); - }); - } else { - done(undefined, storeFileResult); - } - }); - }; - - var updateStorageProperties = function(storeFileResult, done) { - item.storageVerifiedAt = Date.now(); - item.storageFailedAt = undefined; - item.storageBytes = storeFileResult.size; - item.storagePath = storeFileResult.path_lower; - item.save(function(error) { - if (!error) { - debug.success('updateStorageProperties'); - } - - done(error); + var storeFile = function (done) { + debug('storeFile : item.user = ', item.user); + module.exports.storeFile(item.user, item.storage, item.storagePath, data, (error, storeFileResult) => { + if (error) { + debug.error('storeFile item %s, error %o, storeFileResult %o', item.id, error, storeFileResult); + item.storageError = error.message; + item.storageFailedAt = Date.now(); + item.save(() => { + done(error, storeFileResult); }); - }; - - var updateJob = function(done) { - if (job) { - job.incrementTotalItemsStored(); - } + } else { + done(undefined, storeFileResult); + } + }); + }; + + var updateStorageProperties = function (storeFileResult, done) { + item.storageVerifiedAt = Date.now(); + item.storageFailedAt = undefined; + item.storageBytes = storeFileResult.size; + item.storagePath = storeFileResult.path_lower; + item.save(function (error) { + if (!error) { + debug.success('updateStorageProperties'); + } + + done(error); + }); + }; - done(); - }; + var updateJob = function (done) { + if (job) { + job.incrementTotalItemsStored(); + } - var notifyApp = function(done) { - if (app && typeof app.emit === 'function') { - app.emit('storedItemData', item, job); - debug('app notified of storedItemData'); - } else { - debug('app NOT notified of storedItemData'); - } + done(); + }; - done(); - }; + var notifyApp = function (done) { + if (app && typeof app.emit === 'function') { + app.emit('storedItemData', item, job); + debug('app notified of storedItemData'); + } else { + debug('app NOT notified of storedItemData'); + } - async.waterfall([ - validate, - setupLog, - updateStorageAttemptedAt, - storeFile, - updateStorageProperties, - updateJob, - notifyApp - ], function(error) { - if (error) { - log('error', 'Item controller failed to storeItemData', { error: error.message }); - - if (item && item.save) { - item.storageFailedAt = Date.now(); - item.save(function(saveError) { - if (saveError) { - log('error', 'Item controller failed to update item after failure to store it', { error: saveError.message }); - } - - return done(error); - }); - } else { - done(error); - } - } else { - debug.success('storeItemData'); - done(); - } - }); + done(); + }; + + async.waterfall([ + validate, + setupLog, + updateStorageAttemptedAt, + storeFile, + updateStorageProperties, + updateJob, + notifyApp + ], function (error) { + if (error) { + log('error', 'Item controller failed to storeItemData', {error: error.message}); + + if (item && item.save) { + item.storageFailedAt = Date.now(); + item.save(function (saveError) { + if (saveError) { + log('error', 'Item controller failed to update item after failure to store it', {error: saveError.message}); + } + + return done(error); + }); + } else { + done(error); + } + } else { + debug.success('storeItemData'); + done(); + } + }); }; /** @@ -919,105 +919,105 @@ module.exports.storeItemData = function(item, data, job, done) { * @param {Object} data - Object that represents data for file. * @param {function} done - Error-first callback function with object representing HTTP response body from storage request as second parameter. */ -module.exports.storeFile = function(user, storage, path, data, done) { - var log = logger.scopedLog(); - - var validate = function(done) { - validateParams([{ - name: 'user', variable: user, required: true, requiredProperties: ['id'] - }, { - name: 'storage', variable: storage, required: true, requiredProperties: ['id', 'host'] - }, { - name: 'path', variable: path, required: true, requiredType: 'string' - }, { - name: 'data', variable: data, required: true, requiredType: ['buffer', 'object'] - }, { - name: 'done', variable: done, required: true, requiredType: 'function' - }], function(error) { - if (!error) { - var mediaType = mime.lookup(path); - - if (mediaType === 'image/jpeg' && !(data instanceof Buffer)) { - error = new Error('Path parameter with jpg extension not provided with binary data'); - } else if (mediaType === 'application/json' && (data instanceof Buffer)) { - error = new Error('Path parameter with json extension not provided with parseable data'); - } else if (module.exports.hasSupportedMediaType(path) === false) { - error = new Error('Parameter path extension indicates unsupported media type'); - } - } - - done(error); - }); - }; +module.exports.storeFile = function (user, storage, path, data, done) { + var log = logger.scopedLog(); - var prepareData = function(done) { - debug.start('storeFile (path: %s)', path); - if (!(data instanceof Buffer)) { - data = JSON.stringify(data); + var validate = function (done) { + validateParams([{ + name: 'user', variable: user, required: true, requiredProperties: ['id'] + }, { + name: 'storage', variable: storage, required: true, requiredProperties: ['id', 'host'] + }, { + name: 'path', variable: path, required: true, requiredType: 'string' + }, { + name: 'data', variable: data, required: true, requiredType: ['buffer', 'object'] + }, { + name: 'done', variable: done, required: true, requiredType: 'function' + }], function (error) { + if (!error) { + var mediaType = mime.lookup(path); + + if (mediaType === 'image/jpeg' && !(data instanceof Buffer)) { + error = new Error('Path parameter with jpg extension not provided with binary data'); + } else if (mediaType === 'application/json' && (data instanceof Buffer)) { + error = new Error('Path parameter with json extension not provided with parseable data'); + } else if (module.exports.hasSupportedMediaType(path) === false) { + error = new Error('Parameter path extension indicates unsupported media type'); } + } - done(); - }; + done(error); + }); + }; - var setupLog = function(done) { - log = logger.scopedLog({ - path: path, - storage: storage.id, - user: user.id - }); + var prepareData = function (done) { + debug.start('storeFile (path: %s)', path); + if (!(data instanceof Buffer)) { + data = JSON.stringify(data); + } - done(); - }; + done(); + }; - var findUserStorageAuth = function(done) { - UserStorageAuth.findOne({ - storage: storage.id, - user: user.id - }, function(error, userStorageAuth) { - if(!error && !userStorageAuth) { - error = new Error('Failed to retrieve userStorageAuth'); - } + var setupLog = function (done) { + log = logger.scopedLog({ + path: path, + storage: storage.id, + user: user.id + }); - done(error, userStorageAuth); - }); - }; + done(); + }; - var storeFile = function(userStorageAuth, done) { - var options = { - body: data, - headers: storage.headers(path, userStorageAuth), - url: storage.itemPutUrl(path, userStorageAuth) - }; + var findUserStorageAuth = function (done) { + UserStorageAuth.findOne({ + storage: storage.id, + user: user.id + }, function (error, userStorageAuth) { + if (!error && !userStorageAuth) { + error = new Error('Failed to retrieve userStorageAuth'); + } - debug('storeFile:options %o', options); + done(error, userStorageAuth); + }); + }; - // what is going on t is ????? - request.post(options, function(error, res, body) { - if (!error) { - error = request.statusCodeError(res.statusCode); - } + var storeFile = function (userStorageAuth, done) { + var options = { + body: data, + headers: storage.headers(path, userStorageAuth), + url: storage.itemPutUrl(path, userStorageAuth) + }; - if (!error) { - body = JSON.parse(body); - } + debug('storeFile:options %o', options); - debug('storeFile body %o, error %o', body, error); + // what is going on t is ????? + request.post(options, function (error, res, body) { + if (!error) { + error = request.statusCodeError(res.statusCode); + } - done(error, body); - }); - }; + if (!error) { + body = JSON.parse(body); + } - async.waterfall([ - validate, - prepareData, - setupLog, - findUserStorageAuth, - storeFile - ], function(error, responseBody) { - if (error) { - log('error', 'Item controller failed to store file', { error: error.message, responseBody: responseBody }); - } + debug('storeFile body %o, error %o', body, error); - done(error, responseBody); + done(error, body); }); + }; + + async.waterfall([ + validate, + prepareData, + setupLog, + findUserStorageAuth, + storeFile + ], function (error, responseBody) { + if (error) { + log('error', 'Item controller failed to store file', {error: error.message, responseBody: responseBody}); + } + + done(error, responseBody); + }); }; \ No newline at end of file diff --git a/app/lib/jsonapi.js b/app/lib/jsonapi.js index a6d0560..2a5cfcf 100644 --- a/app/lib/jsonapi.js +++ b/app/lib/jsonapi.js @@ -12,11 +12,9 @@ var logger = require('app/lib/logger'); var models = require('app/models'); var ObjectId = require('mongoose').Types.ObjectId; var validateParams = require('./validateParams'); -var tools = require('app/lib/utils/debuggingTools'); const util = require('util'); - var jsonapi = {}; /** @@ -25,7 +23,7 @@ var jsonapi = {}; * @param {Object} document - Mongoose document * @returns {Object} object - JSON API resource object */ -jsonapi.resourceObjectFromDocument = function(document) { +jsonapi.resourceObjectFromDocument = function (document) { if (!document) { throw new Error('No document provided'); } @@ -33,21 +31,21 @@ jsonapi.resourceObjectFromDocument = function(document) { var Model = models[document.modelId()]; var attributes = document.toObject(); - debug("attributes ",util.inspect(attributes)); + debug('attributes ', util.inspect(attributes)); delete attributes.id; var relationships = {}; - Object.keys(attributes).forEach(function(key) { - var addRelationship = function(property) { + Object.keys(attributes).forEach(function (key) { + var addRelationship = function (property) { if (property && property.id && property.modelType) { if (!relationships[key] || !relationships[key].data) { if (Array.isArray(document[key])) { - relationships[key] = { data: [] }; + relationships[key] = {data: []}; } else { - relationships[key] = { data: {} }; + relationships[key] = {data: {}}; } } @@ -94,7 +92,7 @@ jsonapi.resourceObjectFromDocument = function(document) { }); - return { + return { id: document.id, type: document.modelType(), attributes: attributes, @@ -102,7 +100,7 @@ jsonapi.resourceObjectFromDocument = function(document) { }; }; -jsonapi.responseDocument = function(data, included, errors) { +jsonapi.responseDocument = function (data, included, errors) { var doc = {}; if (data) { @@ -128,7 +126,7 @@ jsonapi.responseDocument = function(data, included, errors) { * @param {string} method - HTTP method * @returns {string} allowed value (e.g. "public", "user", or "admin) */ -jsonapi.allowed = function(model, method) { +jsonapi.allowed = function (model, method) { if (typeof model.jsonapi[method] === 'string') { return model.jsonapi[method]; } else if (typeof model.jsonapi[method] === 'object') { @@ -148,7 +146,7 @@ jsonapi.allowed = function(model, method) { */ jsonapi.compiledQueryConditions = function myself(req, conditions, model, method, done) { this.modelQueryConditions(req, model, method, (error, modelConditions) => { - debug("jsonapi.compiledQueryConditions : ",modelConditions,conditions); + debug('jsonapi.compiledQueryConditions : ', modelConditions, conditions); done(error, Object.assign({}, modelConditions, conditions)); }); }; @@ -160,7 +158,7 @@ jsonapi.compiledQueryConditions = function myself(req, conditions, model, method * @param {string} method - HTTP method (optional) * @param {callback} done */ -jsonapi.modelQueryConditions = function(req, model, method, done) { +jsonapi.modelQueryConditions = function (req, model, method, done) { validateParams([{ name: 'req', variable: req, required: true }, { @@ -197,7 +195,7 @@ jsonapi.modelQueryConditions = function(req, model, method, done) { * @param {Object} relationships - Request relationships * @return {Object} Normalized relationships */ -jsonapi.normalizeRelationships = function(relationships) { +jsonapi.normalizeRelationships = function (relationships) { relationships = Object.assign({}, relationships); // Remove any relationships with empty data properties @@ -220,10 +218,10 @@ jsonapi.normalizeRelationships = function(relationships) { * @param {Object} app - Express app * @param {Object} Model - Mongoose model */ -jsonapi.routeModelDeleteObjectResource = function(app, Model) { +jsonapi.routeModelDeleteObjectResource = function (app, Model) { this.routeModelResource(app, Model, 'delete', '/' + _.kebabCase(Model.modelType()) + '/:id', (req, res) => { var getConditions = (done) => { - this.compiledQueryConditions(req, { _id: req.params.id }, Model, 'delete', done); + this.compiledQueryConditions(req, {_id: req.params.id}, Model, 'delete', done); }; var findOne = (conditions, done) => { @@ -236,7 +234,7 @@ jsonapi.routeModelDeleteObjectResource = function(app, Model) { } else if (!document) { this.sendNotFound(res); } else { - document.remove(function(error) { + document.remove(function (error) { if (error) { res.status(500).send(); } else { @@ -253,10 +251,10 @@ jsonapi.routeModelDeleteObjectResource = function(app, Model) { * @param {Object} app - Express app * @param {model} model - Mongoose model */ -jsonapi.routeModelGetObjectResource = function(app, Model) { +jsonapi.routeModelGetObjectResource = function (app, Model) { this.routeModelResource(app, Model, 'get', '/' + _.kebabCase(Model.modelType()) + '/:id', (req, res) => { var getConditions = (done) => { - this.compiledQueryConditions(req, { _id: req.params.id }, Model, 'get', done); + this.compiledQueryConditions(req, {_id: req.params.id}, Model, 'get', done); }; var findOne = (conditions, done) => { @@ -266,7 +264,10 @@ jsonapi.routeModelGetObjectResource = function(app, Model) { async.waterfall([getConditions, findOne], (error, document) => { if (error) { - logger.error('Resource router failed to query for object', { model: Model.modelName, error: error.message }); + logger.error('Resource router failed to query for object', { + model: Model.modelName, + error: error.message + }); this.sendError(res); } else if (!document) { this.sendNotFound(res); @@ -282,12 +283,12 @@ jsonapi.routeModelGetObjectResource = function(app, Model) { * @param {Object} app - Express app * @param {model} model - Mongoose model */ -jsonapi.routeModelGetObjectsResource = function(app, Model) { +jsonapi.routeModelGetObjectsResource = function (app, Model) { this.routeModelResource(app, Model, 'get', '/' + _.kebabCase(Model.modelType()), (req, res) => { var compileConditions = (done) => { var conditions = {}; var filter = req.query.filter ? req.query.filter : {}; - + try { if (filter.relationships) { Object.keys(filter.relationships).forEach((modelName) => { @@ -318,7 +319,7 @@ jsonapi.routeModelGetObjectsResource = function(app, Model) { if (Model.jsonapi.sort) { query.sort(Model.jsonapi.sort); } else { - query.sort({ createdAt: -1 }); + query.sort({createdAt: -1}); } } @@ -330,9 +331,12 @@ jsonapi.routeModelGetObjectsResource = function(app, Model) { query.exec(done); }; - async.waterfall([compileConditions, executeQuery], function(error, documents) { + async.waterfall([compileConditions, executeQuery], function (error, documents) { if (error) { - logger.error('Resource router failed to query for objects', { model: Model.modelName, error: error.message }); + logger.error('Resource router failed to query for objects', { + model: Model.modelName, + error: error.message + }); jsonapi.sendError(res, error, 400); } else { jsonapi.sendDocuments(res, documents); @@ -346,18 +350,18 @@ jsonapi.routeModelGetObjectsResource = function(app, Model) { * @param {Object} app - Express app * @param {Object} Model - Mongoose model */ -jsonapi.routeModelPatchObjectResource = function(app, Model) { +jsonapi.routeModelPatchObjectResource = function (app, Model) { this.routeModelResource(app, Model, 'patch', '/' + _.kebabCase(Model.modelType()) + '/:id', (req, res) => { var validate = (done) => { this.validateQueryData(req, req.body.data, Model, 'patch', done); }; var getConditions = (done) => { - this.compiledQueryConditions(req, { _id: req.params.id }, Model, 'patch', done); + this.compiledQueryConditions(req, {_id: req.params.id}, Model, 'patch', done); }; var findOneAndUpdate = (conditions, done) => { - Model.findOneAndUpdate(conditions, _.camelCasedKeys(req.body.data.attributes), { new: true }, done); + Model.findOneAndUpdate(conditions, _.camelCasedKeys(req.body.data.attributes), {new: true}, done); }; var addRelationships = (document, done) => { @@ -369,7 +373,7 @@ jsonapi.routeModelPatchObjectResource = function(app, Model) { return done(undefined, document); } - this.saveRelationshipsToDocument(document, this.normalizeRelationships(req.body.data.relationships), function(error) { + this.saveRelationshipsToDocument(document, this.normalizeRelationships(req.body.data.relationships), function (error) { done(error, document); }); }; @@ -382,7 +386,7 @@ jsonapi.routeModelPatchObjectResource = function(app, Model) { var executePostRoutine = (document, done) => { if (Model.jsonapi.patch && Model.jsonapi.patch.post) { - Model.jsonapi.patch.post(req, res, document, function(error, req, res, document) { + Model.jsonapi.patch.post(req, res, document, function (error, req, res, document) { done(error, document); }); } else { @@ -421,10 +425,10 @@ jsonapi.routeModelPatchObjectResource = function(app, Model) { * @param {Object} Model - Mongoose model */ -jsonapi.routeModelPostObjectResource = function(app, Model) { -debug ("routeModelPostObjectResource -- Model = ",Model.collection.collectionName); +jsonapi.routeModelPostObjectResource = function (app, Model) { + debug('routeModelPostObjectResource -- Model = ', Model.collection.collectionName); - this.routeModelResource(app, Model, 'post', '/'+ _.kebabCase(Model.modelType()), (req, res) => { + this.routeModelResource(app, Model, 'post', '/' + _.kebabCase(Model.modelType()), (req, res) => { /** * Validates all available attributes (TODO: and relationships) */ @@ -456,7 +460,7 @@ debug ("routeModelPostObjectResource -- Model = ",Model.collection.collectionNam return done(undefined, document); } - this.saveRelationshipsToDocument(document, this.normalizeRelationships(req.body.data.relationships), function(error) { + this.saveRelationshipsToDocument(document, this.normalizeRelationships(req.body.data.relationships), function (error) { done(error, document); }); }; @@ -483,11 +487,11 @@ debug ("routeModelPostObjectResource -- Model = ",Model.collection.collectionNam * Executes any available post-POST routine available for Model */ var executePostRoutine = (document, done) => { - debug("executePostRoutine... ",Model.collection.collectionName); + debug('executePostRoutine... ', Model.collection.collectionName); - if (Model.jsonapi.post && Model.jsonapi.post.post) { + if (Model.jsonapi.post && Model.jsonapi.post.post) { - Model.jsonapi.post.post(req, res, document, function(error) { + Model.jsonapi.post.post(req, res, document, function (error) { done(error, document); }); } else { @@ -524,9 +528,11 @@ debug ("routeModelPostObjectResource -- Model = ",Model.collection.collectionNam * @param {string} path - Path to resource * @param {function} done - Express route callback expecting req and res as parameters */ -jsonapi.routeModelResource = function(app, model, method, path, done) { - debug("jsonapi.routeModelResource -- method = %s, path = %s",method, path); - if (!model.jsonapi || !model.jsonapi[method]) { return; } +jsonapi.routeModelResource = function (app, model, method, path, done) { + debug('jsonapi.routeModelResource -- method = %s, path = %s', method, path); + if (!model.jsonapi || !model.jsonapi[method]) { + return; + } var validateRequestBody = false; @@ -548,7 +554,7 @@ jsonapi.routeModelResource = function(app, model, method, path, done) { * @param {Object} document - Mongoose document * @returns {Object} object - JSON API relationship object */ -jsonapi.resourceIdentifierObjectFromDocument = function(document) { +jsonapi.resourceIdentifierObjectFromDocument = function (document) { if (!document) { throw new Error('No document provided'); } @@ -566,7 +572,7 @@ jsonapi.resourceIdentifierObjectFromDocument = function(document) { * @param {string} name - Name of relationship (e.g. "author") * @param {string} [type=to-many] - Type of relationship (either "to-many" or "to-one") */ -jsonapi.addRelationshipToResourceObject = function(object, document, name, type) { +jsonapi.addRelationshipToResourceObject = function (object, document, name, type) { validateParams([{ name: 'object', variable: object, required: true, requiredType: 'object' }, { @@ -610,7 +616,7 @@ jsonapi.addRelationshipToResourceObject = function(object, document, name, type) * @param {Object} errors - Errors (optional) * @param {number} [status=200] - HTTP status code */ -jsonapi.sendResponseDocument = function(res, data, included, errors, status) { +jsonapi.sendResponseDocument = function (res, data, included, errors, status) { if (!status) { status = 200; } @@ -624,7 +630,7 @@ jsonapi.sendResponseDocument = function(res, data, included, errors, status) { * @param {Object} data – Principal data * @param {Object} included – Included resources (optional) */ -jsonapi.sendData = function(res, data, included) { +jsonapi.sendData = function (res, data, included) { if (!data) { throw new Error('No data parameter provided'); } @@ -638,7 +644,7 @@ jsonapi.sendData = function(res, data, included) { * @param {Object} document - Model document * @param {number} status - HTTP status code */ -jsonapi.sendDocument = function(res, document, status) { +jsonapi.sendDocument = function (res, document, status) { this.sendResponseDocument(res, jsonapi.resourceObjectFromDocument(document), null, null, status); }; @@ -647,7 +653,7 @@ jsonapi.sendDocument = function(res, document, status) { * @param {Object} res – Express response object * @param {Object} documents - Array of model documents */ -jsonapi.sendDocuments = function(res, documents) { +jsonapi.sendDocuments = function (res, documents) { var objects = documents.map((document) => { return jsonapi.resourceObjectFromDocument(document); }); @@ -659,7 +665,7 @@ jsonapi.sendDocuments = function(res, documents) { * Sends response document with 404 status code * @param {Object} res – Express response object */ -jsonapi.sendNotFound = function(res) { +jsonapi.sendNotFound = function (res) { this.sendResponseDocument(res, null, null, null, 404); }; @@ -669,7 +675,7 @@ jsonapi.sendNotFound = function(res) { * @param {Error} error - Error object (optional) with optional errors property * @param {number} [status=500] - HTTP status code */ -jsonapi.sendError = function(res, error, status) { +jsonapi.sendError = function (res, error, status) { if (error) { var errors = error.errors; @@ -678,13 +684,13 @@ jsonapi.sendError = function(res, error, status) { } // Convert object of errors to array if needed - if(typeof errors === 'object' && !Array.isArray(errors)) { - errors = Object.keys(errors).map(function(key) { + if (typeof errors === 'object' && !Array.isArray(errors)) { + errors = Object.keys(errors).map(function (key) { return errors[key]; }); } - errors = errors.map(function(error) { + errors = errors.map(function (error) { return { title: error.message }; @@ -701,13 +707,13 @@ jsonapi.sendError = function(res, error, status) { /** * Establish middleware that generates routes conformant to the JSON API specification for app and Mongoose models */ -jsonapi.routeModelResources = function() { +jsonapi.routeModelResources = function () { /** * Negotiate the Content-Type and Accept request headers * @see {@link http://jsonapi.org/format/#content-negotiation-servers} */ - app.use(function(req, res, next) { - var isModifiedContentType = function(contentType) { + app.use(function (req, res, next) { + var isModifiedContentType = function (contentType) { return (/application\/vnd\.api\+json/.test(contentType) && contentType !== 'application/vnd.api+json'); }; @@ -718,7 +724,7 @@ jsonapi.routeModelResources = function() { if (req.get('Accept')) { var badAccept = false; - req.get('Accept').split(';').forEach(function(accept) { + req.get('Accept').split(';').forEach(function (accept) { badAccept = (isModifiedContentType(accept)); }); @@ -731,7 +737,7 @@ jsonapi.routeModelResources = function() { next(); }); - app.use(function(req, res, next) { + app.use(function (req, res, next) { res.set('Content-Type', 'application/vnd.api+json'); next(); }); @@ -740,7 +746,7 @@ jsonapi.routeModelResources = function() { * Establish body-parser middleware with JSON API error handling */ app.use((req, res, next) => { - var json = bodyParser.json({ type: ['application/vnd.api+json', 'application/json'] }); + var json = bodyParser.json({type: ['application/vnd.api+json', 'application/json']}); json(req, res, (error) => { if (error) { @@ -757,10 +763,10 @@ jsonapi.routeModelResources = function() { }); // Route requests for each model with Mongoose compatability and jsonapi configuration - // that is: set up the routes for specific paths + // that is: set up the routes for specific paths Object.keys(models).forEach((key) => { var model = models[key]; - + if (model.modelName && model.jsonapi) { this.routeModelGetObjectsResource(app, model); this.routeModelGetObjectResource(app, model); @@ -779,7 +785,7 @@ jsonapi.routeModelResources = function() { * @param {Object} middleware - Dictionary of middleware boolean or function values to use for route * @param {function} done - Express route callback expecting req and res as parameters */ -jsonapi.routeResource = function(app, method, path, middleware, done) { +jsonapi.routeResource = function (app, method, path, middleware, done) { //debug("jsonapi.routeResource: ",app, method, path, middleware, done); var requireAuthentication = (req, res, next) => { if (middleware && middleware.requireAuthentication) { @@ -816,8 +822,8 @@ jsonapi.routeResource = function(app, method, path, middleware, done) { * @param {Object} document - Key-value object of relationships * @param {callback} done */ -jsonapi.saveRelationshipsToDocument = function(document, relationships, done) { - var validate = function(done) { +jsonapi.saveRelationshipsToDocument = function (document, relationships, done) { + var validate = function (done) { validateParams([{ name: 'document', variable: document, required: true, requiredType: ['object', 'constructor'] }, { @@ -827,11 +833,11 @@ jsonapi.saveRelationshipsToDocument = function(document, relationships, done) { }], done); }; - var saveRelationshipsToDocument = function(done) { + var saveRelationshipsToDocument = function (done) { var Model = models[document.modelId()]; - - async.forEachOf(relationships, function(relationship, relationshipName, done) { - var validateRelationship = function(done) { + + async.forEachOf(relationships, function (relationship, relationshipName, done) { + var validateRelationship = function (done) { var errors = []; if (!Model.schema.tree[relationshipName]) { @@ -858,9 +864,9 @@ jsonapi.saveRelationshipsToDocument = function(document, relationships, done) { done(error); }; - var addRelationshipToDocument = function(done) { - var validateAndAddResourceIdentifierObjectRelationship = function(resourceObject, done) { - var validateResourceIdentifierObject = function(done) { + var addRelationshipToDocument = function (done) { + var validateAndAddResourceIdentifierObjectRelationship = function (resourceObject, done) { + var validateResourceIdentifierObject = function (done) { var errors = []; if (!resourceObject.id) { @@ -896,7 +902,7 @@ jsonapi.saveRelationshipsToDocument = function(document, relationships, done) { done(error); }; - var addResourceIdentifierToDocument = function(done) { + var addResourceIdentifierToDocument = function (done) { if (Array.isArray(Model.schema.tree[relationshipName]) && document[relationshipName].indexOf(resourceObject.id) === -1) { document[relationshipName].push(resourceObject.id); } else if (!Array.isArray(Model.schema.tree[relationshipName])) { @@ -906,7 +912,7 @@ jsonapi.saveRelationshipsToDocument = function(document, relationships, done) { done(); }; - async.series([validateResourceIdentifierObject, addResourceIdentifierToDocument], function(error) { + async.series([validateResourceIdentifierObject, addResourceIdentifierToDocument], function (error) { done(error, document); }); }; @@ -922,7 +928,7 @@ jsonapi.saveRelationshipsToDocument = function(document, relationships, done) { }, done); }; - async.series([validate, saveRelationshipsToDocument], function(error) { + async.series([validate, saveRelationshipsToDocument], function (error) { done(error, document); }); }; @@ -935,7 +941,7 @@ jsonapi.saveRelationshipsToDocument = function(document, relationships, done) { * @param {string} method - HTTP method (optional) * @param {function} done - Error-first callback function expecting no other parameters */ -jsonapi.validateQueryData = function(req, data, model, method, done) { +jsonapi.validateQueryData = function (req, data, model, method, done) { var getConditions = (done) => { this.modelQueryConditions(req, model, method, done); }; @@ -943,9 +949,9 @@ jsonapi.validateQueryData = function(req, data, model, method, done) { var validateQueryData = (conditions, done) => { var errors = []; - Object.keys(conditions).forEach(function(key) { + Object.keys(conditions).forEach(function (key) { var isEqualObjectId; - + try { isEqualObjectId = ObjectId(_.get(data, `relationships.${key}.data.id`)).equals(conditions[key]); } catch (error) { @@ -974,7 +980,7 @@ jsonapi.validateQueryData = function(req, data, model, method, done) { * Returns Express route middleware that validates request body against JSON API specification and URL for optional model * @param {Object} model - Mongoose model (optional) */ -jsonapi.validateRequestBody = function(model) { +jsonapi.validateRequestBody = function (model) { return (req, res, next) => { var errors = []; diff --git a/app/models/item.js b/app/models/item.js index f9f6ab7..d54b618 100644 --- a/app/models/item.js +++ b/app/models/item.js @@ -9,8 +9,8 @@ var modelFactory = require('app/factories/model'); var queryConditions = require('./queryConditions'); var sanitizeFilename = require('sanitize-filename'); -var convertToFilename = function(content) { - return _.toLower(emojiStrip(sanitizeFilename(content).replace(/[^\x00-\x7F]/g, '').replace('.','').replace('-', ' ').replace(/ {2}/g, ' ').replace(/ +/g, '-').replace(/–|—+/g, '-'))); +var convertToFilename = function (content) { + return _.toLower(emojiStrip(sanitizeFilename(content).replace(/[^\x00-\x7F]/g, '').replace('.', '').replace('-', ' ').replace(/ {2}/g, ' ').replace(/ +/g, '-').replace(/–|—+/g, '-'))); }; /** @@ -28,19 +28,19 @@ var convertToFilename = function(content) { * @property {module:models/user~User} user - User for which item was pulled from source */ module.exports = modelFactory.new('Item', { - contentType: { ref: 'ContentType', required: true }, + contentType: {ref: 'ContentType', required: true}, description: String, - source: { ref: 'Source', required: true }, + source: {ref: 'Source', required: true}, sourceCreatedAt: Date, - sourceItem: { type: String, required: true }, - storage: { ref: 'Storage', required: true }, + sourceItem: {type: String, required: true}, + storage: {ref: 'Storage', required: true}, storageAttemptedAt: Date, storageBytes: Number, storageError: String, storageFailedAt: Date, storagePath: String, storageVerifiedAt: Date, - user: { ref: 'User', required: true } + user: {ref: 'User', required: true} }, { jsonapi: { sort: '-storageVerifiedAt', @@ -50,7 +50,7 @@ module.exports = modelFactory.new('Item', { } } }, { - slug: function(data) { + slug: function (data) { var parts = []; if (data) { diff --git a/app/models/job.js b/app/models/job.js index f5e7ea1..2e67eba 100644 --- a/app/models/job.js +++ b/app/models/job.js @@ -19,20 +19,20 @@ var queryConditions = require('./queryConditions'); * @property {module:models/user~User} [user] - User for which this job should be executed */ module.exports = modelFactory.new('Job', { - contentType: { ref: 'ContentType' }, + contentType: {ref: 'ContentType'}, name: { type: String, required: true, validate: { - validator: function(value) { + validator: function (value) { return (['storeAllItemsForUserStorageSource', 'storeAllItemsForUserStorageSourceContentType'].indexOf(value) > -1); }, message: '"{VALUE}" is not a supported name value' } }, - source: { ref: 'Source' }, - storage: { ref: 'Storage' }, - user: { ref: 'User' }, + source: {ref: 'Source'}, + storage: {ref: 'Storage'}, + user: {ref: 'User'}, totalItemsAvailable: Number, totalItemsStored: Number }, { @@ -65,8 +65,8 @@ module.exports = modelFactory.new('Job', { this.totalItemsAvailable = this.totalItemsAvailable ? total + this.totalItemsAvailable : total; this.save(); } -}, function(schema) { - schema.post('save', function() { +}, function (schema) { + schema.post('save', function () { var itemController = require('app/controllers/item'); var job = this; @@ -74,11 +74,11 @@ module.exports = modelFactory.new('Job', { return; } - var populate = function(done) { + var populate = function (done) { job.populate('contentType source storage user', done); }; - var runJob = function(done) { + var runJob = function (done) { switch (job.name) { case 'storeAllItemsForUserStorageSource': debug('running job "storeAllItemsForUserStorageSource": user %s, source %s, storage %s', job.user.id, job.source.id, job.storage.id); @@ -96,7 +96,7 @@ module.exports = modelFactory.new('Job', { }; async.series([populate, runJob], (error) => { - var log = logger.scopedLog({ jobId: job.id }); + var log = logger.scopedLog({jobId: job.id}); if (error) { debug.error('# job failed: %s', error.message); diff --git a/app/models/source.js b/app/models/source.js index 68691bb..9fe75cc 100644 --- a/app/models/source.js +++ b/app/models/source.js @@ -8,47 +8,45 @@ var nameMethods = require('./methods/name'); // a way to interporlate strings to get a final URL var templateCompiler = require('es6-template-strings'); var SourceContentType = require('app/models/sourceContentType'); -var debug= require('debug'); var methods = Object.assign({ - itemDataObjectsFromPagePath: function(contentType) { - return templateCompiler(this.itemDataObjectsFromPagePathTemplate, { - contentTypePluralCamelName: contentType ? contentType.pluralCamelName() : undefined, - contentTypePluralLowercaseName: contentType ? contentType.pluralLowercaseName() : undefined - }); - }, + itemDataObjectsFromPagePath: function (contentType) { + return templateCompiler(this.itemDataObjectsFromPagePathTemplate, { + contentTypePluralCamelName: contentType ? contentType.pluralCamelName() : undefined, + contentTypePluralLowercaseName: contentType ? contentType.pluralLowercaseName() : undefined + }); + }, - totalItemsAvailableFromPagePath: function(contentType) { - return templateCompiler(this.totalItemsAvailableFromPagePathTemplate, { - contentTypePluralCamelName: contentType ? contentType.pluralCamelName() : undefined, - contentTypePluralLowercaseName: contentType ? contentType.pluralLowercaseName() : undefined - }); - }, + totalItemsAvailableFromPagePath: function (contentType) { + return templateCompiler(this.totalItemsAvailableFromPagePathTemplate, { + contentTypePluralCamelName: contentType ? contentType.pluralCamelName() : undefined, + contentTypePluralLowercaseName: contentType ? contentType.pluralLowercaseName() : undefined + }); + }, - itemsGetUrl: function(itemsGetUrlTemplate,$properties) { + itemsGetUrl: function (itemsGetUrlTemplate, $properties) { - if ($properties.next) { - return $properties.next; - } - console.log("URL to submit == %s", templateCompiler(itemsGetUrlTemplate, $properties)) + if ($properties.next) { + return $properties.next; + } - return templateCompiler(itemsGetUrlTemplate, $properties); - }, + return templateCompiler(itemsGetUrlTemplate, $properties); + }, - /** - * @param done {Function} A callback that will be passed a the error (Error) and optionally the sourceContentTypes results - */ - getSourceContentTypesForSource: function( done){ - SourceContentType.find({source:this.id}, function(err,sourceContentTypes){ - if (err) { - return done(err); - } else { + /** + * @param done {Function} A callback that will be passed a the error (Error) and optionally the sourceContentTypes results + */ + getSourceContentTypesForSource: function (done) { + SourceContentType.find({source: this.id}, function (err, sourceContentTypes) { + if (err) { + return done(err); + } else { - done(err,sourceContentTypes); - } - }); + done(err, sourceContentTypes); + } + }); - } + } }, nameMethods); /** @@ -57,7 +55,6 @@ var methods = Object.assign({ * @property {number} apiVersion - Version of API to use for pulling items from source * @property {string=} clientId - OAuth 2.0 client ID * @property {string=} clientSecret - OAuth 2.0 client secret - * @property {module:models/contentType~ContentType[]} contentTypes - ContentTypes supported by source * @property {boolean} [itemStorageEnabled=false] - Whether source is enabled for storing items in storage * @property {string} [host] - Host URL for source (e.g. "api.foursquare.com") * @property {number} [itemsLimit=25] - Maximum number of items to pull from source in a single page request @@ -69,30 +66,27 @@ var methods = Object.assign({ * @property {string} [totalItemsAvailableFromPagePathTemplate=response.${contentTypePluralCamelName}.count] - String template used to generate object paths to value representing total items available for contentType within pages returned from source */ module.exports = modelFactory.new('Source', { - apiVersion: String, - authScope: Array, - clientId: String, - clientSecret: String, - // contentTypes: [{ ref: 'ContentType' }], - itemDataObjectsFromPagePathTemplate: { type: String, default: 'data' }, - // this is where you specifiy what you want from source - // itemsGetUrlTemplate: { type: String, default: 'https://${host}/${contentTypePluralCamelName}?access_token=${accessToken}&limit=${limit}&offset=${offset}' }, + apiVersion: String, + authScope: Array, + clientId: String, + clientSecret: String, + itemDataObjectsFromPagePathTemplate: {type: String, default: 'data'}, - itemStorageEnabled: { type: Boolean, default: false }, - host: String, - itemsLimit: { type: Number, default: 25 }, - logoGlyphPath: String, - name: { type: String, required: true }, - passportStrategy: String, - slug: String, - totalItemsAvailableFromPagePathTemplate: String + itemStorageEnabled: {type: Boolean, default: false}, + host: String, + itemsLimit: {type: Number, default: 25}, + logoGlyphPath: String, + name: {type: String, required: true}, + passportStrategy: String, + slug: String, + totalItemsAvailableFromPagePathTemplate: String }, { - jsonapi: { - delete: 'admin', - filteredProperties: ['clientId', 'clientSecret'], - get: 'public', - patch: 'admin', - post: 'admin' - } + jsonapi: { + delete: 'admin', + filteredProperties: ['clientId', 'clientSecret'], + get: 'public', + patch: 'admin', + post: 'admin' + } }, methods); \ No newline at end of file diff --git a/app/models/sourceContentType.js b/app/models/sourceContentType.js index 7bb06b9..cc3818a 100644 --- a/app/models/sourceContentType.js +++ b/app/models/sourceContentType.js @@ -15,14 +15,14 @@ var nameMethods = require('./methods/name'); * @property {module:models/contentType~ContentType} contentType for this sourceContentType */ module.exports = modelFactory.new('SourceContentType', { - source: { ref: 'Source',required: true }, - contentType: { ref: 'ContentType',required: true }, - itemsGetUrlTemplate:{ type:String, default: "https://${sourceHost}/${contentTypePluralCamelName}?access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}"} + source: { ref: 'Source',required: true }, + contentType: { ref: 'ContentType',required: true }, + itemsGetUrlTemplate:{ type:String, default: 'https://${sourceHost}/${contentTypePluralCamelName}?access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}'} }, { - jsonapi: { - get: 'public', - post: 'public', - del: 'public' - } + jsonapi: { + get: 'public', + post: 'public', + del: 'public' + } }, nameMethods); \ No newline at end of file diff --git a/app/models/userSourceAuth.js b/app/models/userSourceAuth.js index 0e2957c..db601e5 100644 --- a/app/models/userSourceAuth.js +++ b/app/models/userSourceAuth.js @@ -15,10 +15,10 @@ var queryConditions = require('./queryConditions'); * @property {module:models/user~User} [user] - User to authenticate at source */ module.exports = modelFactory.new('UserSourceAuth', { - source: { ref: 'Source', required: true }, + source: {ref: 'Source', required: true}, sourceToken: String, sourceUser: String, - user: { ref: 'User' } + user: {ref: 'User'} }, { jsonapi: { get: { diff --git a/tests/controllers/item/itemsGetUrl.js b/tests/controllers/item/itemsGetUrl.js index 8707d49..f5e9b37 100644 --- a/tests/controllers/item/itemsGetUrl.js +++ b/tests/controllers/item/itemsGetUrl.js @@ -7,8 +7,8 @@ var wh = require('app/lib/warehouse'); describe('itemController.itemsGetUrl method', function() { assertions.function.returnsResult('controller.itemsGetUrl', controller.itemsGetUrl, [{ - when: 'provided source (with itemsGetUrlTemplate property), contentType, userSourceAuth, and pagination as parameters', - params: [wh.one('source'), wh.one('contentType'), wh.one('userSourceAuth'), wh.pagination()], + when: 'provided source (with itemsGetUrlTemplate property), sourceContentType, userSourceAuth, and pagination as parameters', + params: [wh.one('source'), wh.one('sourceContentType'), wh.one('userSourceAuth'), wh.pagination()], result: function(itemsGetUrl, done) { assert.equal(itemsGetUrl, templateCompiler(this.params[0].itemsGetUrlTemplate, { host: this.params[0].host, @@ -20,8 +20,8 @@ describe('itemController.itemsGetUrl method', function() { done(); } }, { - when: 'provided source (without itemsGetUrlTemplate property), contentType, userSourceAuth, and pagination as parameters', - params: [wh.one('source'), wh.one('contentType'), wh.one('userSourceAuth'), wh.pagination()], + when: 'provided source (without itemsGetUrlTemplate property), sourceContentType, userSourceAuth, and pagination as parameters', + params: [wh.one('source'), wh.one('sourceContentType'), wh.one('userSourceAuth'), wh.pagination()], result: function(itemsGetUrl, done) { assert.equal(itemsGetUrl, templateCompiler('https://${host}/${contentTypePluralCamelName}?access_token=${accessToken}&limit=${limit}&offset=${offset}', { host: this.params[0].host, @@ -43,27 +43,27 @@ describe('itemController.itemsGetUrl method', function() { assertions.function.throws.error('controller.itemsGetUrl', controller.itemsGetUrl, [{ when: 'no source parameter provided', - params: [undefined, wh.one('contentType'), wh.one('userSourceAuth'), wh.pagination()], + params: [undefined, wh.one('sourceContentType'), wh.one('userSourceAuth'), wh.pagination()], error: 'Parameter source undefined or null' }, { when: 'source parameter has no host property', - params: [{}, wh.one('contentType'), wh.one('userSourceAuth'), wh.pagination()], + params: [{}, wh.one('sourceContentType'), wh.one('userSourceAuth'), wh.pagination()], error: 'Parameter source has no host property' }, { - when: 'no contentType parameter provided', + when: 'no sourceContentType parameter provided', params: [wh.one('source'), undefined, wh.one('userSourceAuth'), wh.pagination()], - error: 'Parameter contentType undefined or null' + error: 'Parameter sourceContentType undefined or null' }, { - when: 'contentType parameter has no name property', + when: 'sourceContentType parameter has no name property', params: [wh.one('source'), {}, wh.one('userSourceAuth'), wh.pagination()], - error: 'Parameter contentType has no name property' + error: 'Parameter sourceContentType has no name property' }, { when: 'no userSourceAuth parameter provided', - params: [wh.one('source'), wh.one('contentType'), undefined, wh.pagination()], + params: [wh.one('source'), wh.one('sourceContentType'), undefined, wh.pagination()], error: 'Parameter userSourceAuth undefined or null' }, { when: 'userSourceAuth parameter has no sourceToken property', - params: [wh.one('source'), wh.one('contentType'), {}, wh.pagination()], + params: [wh.one('source'), wh.one('sourceContentType'), {}, wh.pagination()], error: 'Parameter userSourceAuth has no sourceToken property' }]); }); \ No newline at end of file From 5b87b611d3413b60755e798fd4c67e96a11b3891 Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Mon, 21 Aug 2017 18:10:37 +0200 Subject: [PATCH 05/14] Edit .eslintrc.js to implement new ESLint rules (prior to making further changes in source files) --- .eslintrc.js | 31 +++++++++++++++++++++++++++++++ fixtures/models.js | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.eslintrc.js b/.eslintrc.js index 5a21701..3793fa3 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -32,6 +32,37 @@ module.exports = { "semi": [ "warn", "always" + ], + "key-spacing": [ + "warn", + "always" + ], + "func-call-spacing": [ + "warn", + "always" + ], "block-spacing": [ + "warn", + "always" + ], + "array-bracket-spacing": [ + "warn", + "always" + ], "keyword-spacing": [ + "warn", + "always" + ], + "semi-spacing": [ + "warn", + "always" + ], "arrow-spacing": [ + "warn", + "always" + ], + "no-tabs": [ + "warn", + "always" ] + + } }; \ No newline at end of file diff --git a/fixtures/models.js b/fixtures/models.js index 3bf44f0..b1cd34a 100644 --- a/fixtures/models.js +++ b/fixtures/models.js @@ -292,7 +292,7 @@ module.exports = { post: 'admin' }, mockProperties: () => { - return { + return { _id: ObjectId(), apiVersion: '99', authScope: ['foo','bar'], From 8f10be7fff37c438c6549ee58f286d9f32ffc95f Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Mon, 21 Aug 2017 19:17:21 +0200 Subject: [PATCH 06/14] Edit .eslintrc.js to implement new ESLint rules (add /space-before-function-paren); fix errors that ESLint detected --- .eslintrc.js | 29 ++++++++++------------ app/controllers/item.js | 4 +-- app/lib/jsonapi.js | 2 +- app/lib/mongoose.js | 2 +- app/lib/utils/debuggingTools | 41 +++++++++++++++---------------- app/lib/warehouse.js | 2 +- app/models/item.js | 4 +-- app/models/job.js | 10 ++++---- app/models/notificationRequest.js | 2 +- app/models/source.js | 10 ++++---- app/models/sourceContentType.js | 2 +- 11 files changed, 52 insertions(+), 56 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 3793fa3..87eae32 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -34,33 +34,30 @@ module.exports = { "always" ], "key-spacing": [ - "warn", - "always" + "warn" ], "func-call-spacing": [ - "warn", - "always" + "error", + "never" ], "block-spacing": [ - "warn", - "always" + "error" ], "array-bracket-spacing": [ - "warn", - "always" + "error" ], "keyword-spacing": [ - "warn", - "always" + "error" ], "semi-spacing": [ - "warn", - "always" + "error" ], "arrow-spacing": [ - "warn", - "always" + "error" ], "no-tabs": [ - "warn", - "always" + "error" + ], + "space-before-function-paren": [ + "error", + "never" ] diff --git a/app/controllers/item.js b/app/controllers/item.js index b45fc3e..45277ba 100644 --- a/app/controllers/item.js +++ b/app/controllers/item.js @@ -629,7 +629,7 @@ module.exports.storeItemsPage = function (user, source, storage, sourceContentTy if (job) { jobAttributes.jobId = job.id; } -// this is where the actual jobs queue "items" are created + // this is where the actual jobs queue "items" are created /// where the magic happens… var queueJob = queue.create(STORE_ITEM_DATA, jobAttributes).save((error) => { if (error) { @@ -830,7 +830,7 @@ module.exports.storeItemData = function (item, data, job, done) { done(error); }); }; -// ?? what is going on here?? + // ?? what is going on here?? var storeFile = function (done) { debug('storeFile : item.user = ', item.user); module.exports.storeFile(item.user, item.storage, item.storagePath, data, (error, storeFileResult) => { diff --git a/app/lib/jsonapi.js b/app/lib/jsonapi.js index 2a5cfcf..471ebea 100644 --- a/app/lib/jsonapi.js +++ b/app/lib/jsonapi.js @@ -757,7 +757,7 @@ jsonapi.routeModelResources = function () { }); }); -// default response + // default response app.get('/', (req, res) => { this.sendResponseDocument(res); }); diff --git a/app/lib/mongoose.js b/app/lib/mongoose.js index 8d796fa..822a404 100644 --- a/app/lib/mongoose.js +++ b/app/lib/mongoose.js @@ -33,7 +33,7 @@ mongoose.transform = function(doc, ret) { } // Convert under_scores to camelCase - newKey = key.replace(/_([a-z])/g, function (g) { return g[1].toUpperCase(); }); + newKey = key.replace(/_([a-z])/g, function(g) { return g[1].toUpperCase(); }); if (newKey != key) { ret[newKey] = ret[key]; diff --git a/app/lib/utils/debuggingTools b/app/lib/utils/debuggingTools index db0b3b9..996e971 100644 --- a/app/lib/utils/debuggingTools +++ b/app/lib/utils/debuggingTools @@ -11,38 +11,37 @@ var debug = require('debug')('syncServer:debuggerTools'); debugTools.noAsyncTrace= function(depth=1,compact=false,filter='async.js'){ - this.stackTrace(depth,compact,filter); -} + this.stackTrace(depth,compact,filter); +}; debugTools.stackTrace = function(depth = 1,compact=false, filter=null) { - depth = depth < 1 ? 1 : depth; - let theCurrentStack = sTrace.get(); - let tmp,fileName, connector,output; + depth = depth < 1 ? 1 : depth; + let theCurrentStack = sTrace.get(); + let tmp,fileName,output; - output = "trace: "; - connector = compact ? ' -> ' : '/n'; - depth = depth < theCurrentStack.length -1 ? depth : theCurrentStack.length -1; + output = 'trace: '; + depth = depth < theCurrentStack.length -1 ? depth : theCurrentStack.length -1; - for (let i = 1; i <= depth; i++) { - tmp = theCurrentStack[i]; - fileName = tmp.getFileName(); + for (let i = 1; i <= depth; i++) { + tmp = theCurrentStack[i]; + fileName = tmp.getFileName(); - if (fileName.indexOf('/') > -1) { - fileName = fileName.split('/').pop(); - if (filter && filter.indexOf(fileName) > -1) continue; - } + if (fileName.indexOf('/') > -1) { + fileName = fileName.split('/').pop(); + if (filter && filter.indexOf(fileName) > -1) continue; + } - if (compact) { - output += `${fileName}.${tmp.getFunctionName()}, --> `; - } else { + if (compact) { + output += `${fileName}.${tmp.getFunctionName()}, --> `; + } else { - output += `${tmp.getFunctionName()}, ${fileName}, ${tmp.getLineNumber()}\n`; - } + output += `${tmp.getFunctionName()}, ${fileName}, ${tmp.getLineNumber()}\n`; } + } - debug(output); + debug(output); }; diff --git a/app/lib/warehouse.js b/app/lib/warehouse.js index 886e9bf..c949b10 100644 --- a/app/lib/warehouse.js +++ b/app/lib/warehouse.js @@ -206,7 +206,7 @@ module.exports = { one: function(modelId, overwriteProperties) { if (ObjectId.isValid(overwriteProperties)) { overwriteProperties = { - _id : overwriteProperties + _id: overwriteProperties }; } diff --git a/app/models/item.js b/app/models/item.js index d54b618..6cce6c8 100644 --- a/app/models/item.js +++ b/app/models/item.js @@ -9,7 +9,7 @@ var modelFactory = require('app/factories/model'); var queryConditions = require('./queryConditions'); var sanitizeFilename = require('sanitize-filename'); -var convertToFilename = function (content) { +var convertToFilename = function(content) { return _.toLower(emojiStrip(sanitizeFilename(content).replace(/[^\x00-\x7F]/g, '').replace('.', '').replace('-', ' ').replace(/ {2}/g, ' ').replace(/ +/g, '-').replace(/–|—+/g, '-'))); }; @@ -50,7 +50,7 @@ module.exports = modelFactory.new('Item', { } } }, { - slug: function (data) { + slug: function(data) { var parts = []; if (data) { diff --git a/app/models/job.js b/app/models/job.js index 2e67eba..fd4d2cc 100644 --- a/app/models/job.js +++ b/app/models/job.js @@ -24,7 +24,7 @@ module.exports = modelFactory.new('Job', { type: String, required: true, validate: { - validator: function (value) { + validator: function(value) { return (['storeAllItemsForUserStorageSource', 'storeAllItemsForUserStorageSourceContentType'].indexOf(value) > -1); }, message: '"{VALUE}" is not a supported name value' @@ -65,8 +65,8 @@ module.exports = modelFactory.new('Job', { this.totalItemsAvailable = this.totalItemsAvailable ? total + this.totalItemsAvailable : total; this.save(); } -}, function (schema) { - schema.post('save', function () { +}, function(schema) { + schema.post('save', function() { var itemController = require('app/controllers/item'); var job = this; @@ -74,11 +74,11 @@ module.exports = modelFactory.new('Job', { return; } - var populate = function (done) { + var populate = function(done) { job.populate('contentType source storage user', done); }; - var runJob = function (done) { + var runJob = function(done) { switch (job.name) { case 'storeAllItemsForUserStorageSource': debug('running job "storeAllItemsForUserStorageSource": user %s, source %s, storage %s', job.user.id, job.source.id, job.storage.id); diff --git a/app/models/notificationRequest.js b/app/models/notificationRequest.js index 0fd65b2..745d509 100644 --- a/app/models/notificationRequest.js +++ b/app/models/notificationRequest.js @@ -1,4 +1,4 @@ - /** +/** * NotificationRequest model * @module */ diff --git a/app/models/source.js b/app/models/source.js index 9fe75cc..80bcdd5 100644 --- a/app/models/source.js +++ b/app/models/source.js @@ -10,21 +10,21 @@ var templateCompiler = require('es6-template-strings'); var SourceContentType = require('app/models/sourceContentType'); var methods = Object.assign({ - itemDataObjectsFromPagePath: function (contentType) { + itemDataObjectsFromPagePath: function(contentType) { return templateCompiler(this.itemDataObjectsFromPagePathTemplate, { contentTypePluralCamelName: contentType ? contentType.pluralCamelName() : undefined, contentTypePluralLowercaseName: contentType ? contentType.pluralLowercaseName() : undefined }); }, - totalItemsAvailableFromPagePath: function (contentType) { + totalItemsAvailableFromPagePath: function(contentType) { return templateCompiler(this.totalItemsAvailableFromPagePathTemplate, { contentTypePluralCamelName: contentType ? contentType.pluralCamelName() : undefined, contentTypePluralLowercaseName: contentType ? contentType.pluralLowercaseName() : undefined }); }, - itemsGetUrl: function (itemsGetUrlTemplate, $properties) { + itemsGetUrl: function(itemsGetUrlTemplate, $properties) { if ($properties.next) { return $properties.next; @@ -36,8 +36,8 @@ var methods = Object.assign({ /** * @param done {Function} A callback that will be passed a the error (Error) and optionally the sourceContentTypes results */ - getSourceContentTypesForSource: function (done) { - SourceContentType.find({source: this.id}, function (err, sourceContentTypes) { + getSourceContentTypesForSource: function(done) { + SourceContentType.find({source: this.id}, function(err, sourceContentTypes) { if (err) { return done(err); } else { diff --git a/app/models/sourceContentType.js b/app/models/sourceContentType.js index cc3818a..3269513 100644 --- a/app/models/sourceContentType.js +++ b/app/models/sourceContentType.js @@ -17,7 +17,7 @@ var nameMethods = require('./methods/name'); module.exports = modelFactory.new('SourceContentType', { source: { ref: 'Source',required: true }, contentType: { ref: 'ContentType',required: true }, - itemsGetUrlTemplate:{ type:String, default: 'https://${sourceHost}/${contentTypePluralCamelName}?access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}'} + itemsGetUrlTemplate: { type: String, default: 'https://${sourceHost}/${contentTypePluralCamelName}?access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}'} }, { jsonapi: { From 5fe7233fef1da7f835ba7921f3f8bd0168b31b1c Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Mon, 21 Aug 2017 19:23:44 +0200 Subject: [PATCH 07/14] Further correct ESLint errors (controllers/item.js, lib/jsonapi.js) --- app/controllers/item.js | 154 ++++++++++++++++++++-------------------- app/lib/jsonapi.js | 98 ++++++++++++------------- 2 files changed, 126 insertions(+), 126 deletions(-) diff --git a/app/controllers/item.js b/app/controllers/item.js index 45277ba..a4596de 100644 --- a/app/controllers/item.js +++ b/app/controllers/item.js @@ -37,7 +37,7 @@ var queue = kue.createQueue(); const STORE_ITEM_DATA = 'storeItemData'; -queue.process(STORE_ITEM_DATA, function (queueJob, done) { +queue.process(STORE_ITEM_DATA, function(queueJob, done) { debug('process queueJob %s', queueJob.id); var getItem = (done) => { @@ -90,13 +90,13 @@ queue.on('error', (error) => { * @param {string} url - URL of resource with extension that corresponds to a supported media type. * @param {module:controllers/item~resourceCallback} done */ -module.exports.getResource = function (url, done) { +module.exports.getResource = function(url, done) { var log = logger.scopedLog(); - var validate = function (done) { + var validate = function(done) { validateParams([{ name: 'url', variable: url, required: true, requiredType: 'string', regex: urlRegex - }], function (error) { + }], function(error) { if (!error && module.exports.hasSupportedMediaType(url) === false) { error = new Error('Parameter url indicates unsupported media type'); } @@ -105,7 +105,7 @@ module.exports.getResource = function (url, done) { }); }; - var setupLog = function (done) { + var setupLog = function(done) { debug.start('getResource %s', url); log = logger.scopedLog({ @@ -115,7 +115,7 @@ module.exports.getResource = function (url, done) { done(); }; - var getResource = function (done) { + var getResource = function(done) { var mediaType = mime.lookup(url); mediaType = mediaType ? mediaType : 'application/json'; @@ -124,7 +124,7 @@ module.exports.getResource = function (url, done) { headers: { 'Content-Type': mediaType } - }, function (error, res, body) { + }, function(error, res, body) { if (error) { return done(error); } else if (request.statusCodeError(res.statusCode)) { @@ -157,7 +157,7 @@ module.exports.getResource = function (url, done) { validate, setupLog, getResource - ], function (error, resource) { + ], function(error, resource) { if (error) { log('error', 'Item controller failed to get resource', {error: error}); } @@ -171,7 +171,7 @@ module.exports.getResource = function (url, done) { * @param {string} url - URL * @returns {boolean|undefined} Whether media type supported by controller operations */ -module.exports.hasSupportedMediaType = function (url) { +module.exports.hasSupportedMediaType = function(url) { validateParams([{ name: 'url', variable: url, required: true, requiredType: 'string' }]); @@ -195,7 +195,7 @@ module.exports.hasSupportedMediaType = function (url) { * @param {ContentType} contentType * @returns {Object[]} ItemDataObjects */ -module.exports.itemDataObjectsFromPage = function (page, source, contentType) { +module.exports.itemDataObjectsFromPage = function(page, source, contentType) { validateParams([{ name: 'page', variable: page, required: true, requiredType: 'object' }, { @@ -221,7 +221,7 @@ module.exports.itemDataObjectsFromPage = function (page, source, contentType) { * @param {Object} pagination - Pagination used to make request. * @returns {string} URL for making a GET request */ -module.exports.itemsGetUrl = function (source, sourceContentType, userSourceAuth, pagination) { +module.exports.itemsGetUrl = function(source, sourceContentType, userSourceAuth, pagination) { validateParams([{ name: 'source', variable: source, required: true, requiredProperties: ['host'] }, { @@ -261,7 +261,7 @@ module.exports.itemsGetUrl = function (source, sourceContentType, userSourceAuth * @param {Object} contentType - ContentType of items. * @returns {number} Total number of items available. */ -module.exports.totalItemsAvailableFromPage = function (page, source, contentType) { +module.exports.totalItemsAvailableFromPage = function(page, source, contentType) { validateParams([{ name: 'page', variable: page, required: true, requiredType: 'object' }, { @@ -285,7 +285,7 @@ module.exports.totalItemsAvailableFromPage = function (page, source, contentType * @param {Object} page - Items page. * @returns {error} Error */ -module.exports.itemsPageError = function (page) { +module.exports.itemsPageError = function(page) { validateParams([{ name: 'page', variable: page, required: true, requiredType: 'object' }]); @@ -313,7 +313,7 @@ module.exports.itemsPageError = function (page) { * @param {Object} contentType - ContentType of current items page. * @returns {Object} Pagination for next items page. */ -module.exports.itemsPageNextPagination = function (page, pagination, contentType) { +module.exports.itemsPageNextPagination = function(page, pagination, contentType) { validateParams([{ name: 'page', variable: page, required: true, requiredType: 'object' }, { @@ -355,14 +355,14 @@ module.exports.itemsPageNextPagination = function (page, pagination, contentType * @param {Object} data - Raw item data from source. * @param {function} done - Error-first callback function expecting file system path as second parameter. */ -module.exports.storagePath = function (item, data, done) { - var validate = function (done) { +module.exports.storagePath = function(item, data, done) { + var validate = function(done) { validateParams([{ name: 'item', variable: item, required: true, requiredProperties: ['id', 'contentType'] }], done); }; - var storagePath = function (done) { + var storagePath = function(done) { var path = '/' + item.source.kebabName() + '/' + item.contentType.pluralKebabName() + '/' + item.slug(data) + '.json'; debug.success('storagePath: %s', path); done(undefined, path); @@ -381,10 +381,10 @@ module.exports.storagePath = function (item, data, done) { * @param {Job} [job] - Job for which to store items. * @param {callback} done */ -module.exports.storeAllForUserStorageSource = function (user, source, storage, job, done) { +module.exports.storeAllForUserStorageSource = function(user, source, storage, job, done) { var log = logger.scopedLog(); - var validate = function (done) { + var validate = function(done) { validateParams([{ name: 'user', variable: user, required: true, requiredProperties: ['id'] }, { @@ -394,7 +394,7 @@ module.exports.storeAllForUserStorageSource = function (user, source, storage, j }], done); }; - var setupLog = function (done) { + var setupLog = function(done) { debug.start('storeAllForUserStorageSource'); log = logger.scopedLog({ @@ -406,14 +406,14 @@ module.exports.storeAllForUserStorageSource = function (user, source, storage, j done(); }; - var storeAllForUserStorageSourceContentType = function (sourceContentType, done) { + var storeAllForUserStorageSourceContentType = function(sourceContentType, done) { module.exports.storeAllForUserStorageSourceContentType(user, source, storage, sourceContentType, job, done); }; - let getSourceContentTypesFunction = function (done) { + let getSourceContentTypesFunction = function(done) { - source.getSourceContentTypesForSource(function (err, sourceContentTypes) { + source.getSourceContentTypesForSource(function(err, sourceContentTypes) { if (err) { return done(err); } else { @@ -422,13 +422,13 @@ module.exports.storeAllForUserStorageSource = function (user, source, storage, j }); }; - var storeAllItems = function (sourceContentTypes, done) { + var storeAllItems = function(sourceContentTypes, done) { debug.start('storeAllItems (sourceContentTypes: %s)', sourceContentTypes.length); async.eachSeries(sourceContentTypes, storeAllForUserStorageSourceContentType, done); }; - async.waterfall([validate, setupLog, getSourceContentTypesFunction, storeAllItems], function (error) { + async.waterfall([validate, setupLog, getSourceContentTypesFunction, storeAllItems], function(error) { if (error) { log('error', 'Item controller failed to store all items', {error: error.message}); } else { @@ -450,11 +450,11 @@ module.exports.storeAllForUserStorageSource = function (user, source, storage, j * @param {Job} [job] - Job for which to store items. * @param {callback} done */ -module.exports.storeAllForUserStorageSourceContentType = function (user, source, storage, sourceContentType, job, done) { +module.exports.storeAllForUserStorageSourceContentType = function(user, source, storage, sourceContentType, job, done) { debug('storeAllForUserStorageSourceContentType, source = %s, storage = %s, sourceContentType = %s', source, storage, sourceContentType); var log = logger.scopedLog(); - var validate = function (done) { + var validate = function(done) { validateParams([{ name: 'user', variable: user, required: true, requiredProperties: ['id'] }, { @@ -466,7 +466,7 @@ module.exports.storeAllForUserStorageSourceContentType = function (user, source, }], done); }; - var setupLog = function (done) { + var setupLog = function(done) { debug.start('storeAllForUserStorageSourceContentType'); log = logger.scopedLog({ @@ -479,7 +479,7 @@ module.exports.storeAllForUserStorageSourceContentType = function (user, source, done(); }; - var storeAllItems = function (done) { + var storeAllItems = function(done) { var storeAllItemPages = function myself(error, pagination) { if (error) { if (done) { @@ -497,7 +497,7 @@ module.exports.storeAllForUserStorageSourceContentType = function (user, source, storeAllItemPages(null, {offset: 0}); }; - async.series([validate, setupLog, storeAllItems], function (error) { + async.series([validate, setupLog, storeAllItems], function(error) { if (error) { debug.error('storeAllForUserStorageSourceContentType (message: %s)', error.message); log('error', 'Item controller failed to store all items', {error: error}); @@ -522,12 +522,12 @@ module.exports.storeAllForUserStorageSourceContentType = function (user, source, * @param {Job} [job] - Job for which to store items. * @param {callback} done */ -module.exports.storeItemsPage = function (user, source, storage, sourceContentType, pagination, job, done) { +module.exports.storeItemsPage = function(user, source, storage, sourceContentType, pagination, job, done) { var log = logger.scopedLog(); var ids, page, userSourceAuth; - var validate = function (done) { + var validate = function(done) { validateParams([{ name: 'user', variable: user, required: true, requiredProperties: ['id'] }, { @@ -541,7 +541,7 @@ module.exports.storeItemsPage = function (user, source, storage, sourceContentTy }], done); }; - var setupLog = function (done) { + var setupLog = function(done) { debug.start('## storeItemsPage (sourceContentType: %s, pagination: %o)', sourceContentType.id, pagination); ids = { @@ -555,11 +555,11 @@ module.exports.storeItemsPage = function (user, source, storage, sourceContentTy done(); }; - var findUserSourceAuth = function (done) { + var findUserSourceAuth = function(done) { UserSourceAuth.findOne({ user: user.id, source: source.id - }, function (error, foundUserSourceAuth) { + }, function(error, foundUserSourceAuth) { if (!foundUserSourceAuth && !error) { error = new Error('Failed to find userSourceAuth'); } @@ -572,11 +572,11 @@ module.exports.storeItemsPage = function (user, source, storage, sourceContentTy // get the page of date from URL // whcih will be converted INTO items - var getItemsPageResource = function (done) { + var getItemsPageResource = function(done) { module.exports.getResource(module.exports.itemsGetUrl(source, sourceContentType, userSourceAuth, pagination), done); }; - var getItemDataObjects = function (resource, done) { + var getItemDataObjects = function(resource, done) { page = resource; var error = module.exports.itemsPageError(page); @@ -598,9 +598,9 @@ module.exports.storeItemsPage = function (user, source, storage, sourceContentTy done(undefined, itemDataObjects); }; - var persistItemDataObjects = function (itemDataObjects, done) { + var persistItemDataObjects = function(itemDataObjects, done) { var count = 0; - async.mapSeries(itemDataObjects, function (itemDataObject, done) { + async.mapSeries(itemDataObjects, function(itemDataObject, done) { count++; debug('persistItemDataObject #%s', count); @@ -619,8 +619,8 @@ module.exports.storeItemsPage = function (user, source, storage, sourceContentTy }, done); }; - var storeItemsData = function (itemPairs, done) { - async.each(itemPairs, function (itemPair, done) { + var storeItemsData = function(itemPairs, done) { + async.each(itemPairs, function(itemPair, done) { var jobAttributes = { itemId: itemPair.item.id, data: itemPair.data @@ -643,7 +643,7 @@ module.exports.storeItemsPage = function (user, source, storage, sourceContentTy }, done); }; - var determineNextPagination = function (done) { + var determineNextPagination = function(done) { done(undefined, module.exports.itemsPageNextPagination(page, pagination, sourceContentType.contentType)); }; @@ -656,7 +656,7 @@ module.exports.storeItemsPage = function (user, source, storage, sourceContentTy persistItemDataObjects, storeItemsData, determineNextPagination - ], function (error, nextPagination) { + ], function(error, nextPagination) { if (error) { log('error', 'Item controller failed to store page of items', {error: error.message}); } else { @@ -677,11 +677,11 @@ module.exports.storeItemsPage = function (user, source, storage, sourceContentTy */ // this creates the Item _about_ the data we're about to store, and saves this MongoDBx -module.exports.persistItemDataObject = function (itemDataObject, relationships, done) { +module.exports.persistItemDataObject = function(itemDataObject, relationships, done) { var conditions; var log = logger.scopedLog(); - var validate = function (done) { + var validate = function(done) { validateParams([{ name: 'itemDataObject', variable: itemDataObject, required: true, requiredProperties: ['id'] }, { @@ -692,7 +692,7 @@ module.exports.persistItemDataObject = function (itemDataObject, relationships, }], done); }; - var compileConditions = function (done) { + var compileConditions = function(done) { debug.start('persistItemDataObject'); conditions = { @@ -705,13 +705,13 @@ module.exports.persistItemDataObject = function (itemDataObject, relationships, done(); }; - var setupLog = function (done) { + var setupLog = function(done) { log = logger.scopedLog(conditions); done(); }; - var persistItemDataObject = function (done) { - Item.findOrCreate(conditions, function (error, item) { + var persistItemDataObject = function(done) { + Item.findOrCreate(conditions, function(error, item) { if (error) { done(error); } else { @@ -720,7 +720,7 @@ module.exports.persistItemDataObject = function (itemDataObject, relationships, }); }; - var saveSourceCreatedAt = function (item, done) { + var saveSourceCreatedAt = function(item, done) { var createdAt = itemDataObject.createdAt ? itemDataObject.createdAt * 1000 : null; createdAt = !createdAt && itemDataObject.created_time ? itemDataObject.created_time : createdAt; @@ -734,7 +734,7 @@ module.exports.persistItemDataObject = function (itemDataObject, relationships, } }; - var saveDescription = function (item, done) { + var saveDescription = function(item, done) { if (itemDataObject) { var parts = []; @@ -763,13 +763,13 @@ module.exports.persistItemDataObject = function (itemDataObject, relationships, } }; - var determinePath = function (item, done) { - module.exports.storagePath(item, itemDataObject, function (error, path) { + var determinePath = function(item, done) { + module.exports.storagePath(item, itemDataObject, function(error, path) { done(error, path, item); }); }; - var savePath = function (path, item, done) { + var savePath = function(path, item, done) { item.storagePath = path; item.save((error) => { done(error, item); @@ -785,7 +785,7 @@ module.exports.persistItemDataObject = function (itemDataObject, relationships, saveDescription, determinePath, savePath - ], function (error, item) { + ], function(error, item) { if (error) { log('error', 'Item controller failed to persist item data object', {error: error}); } else { @@ -807,10 +807,10 @@ module.exports.persistItemDataObject = function (itemDataObject, relationships, * @param {callback} done */ // item is newly created item, data is the data for that item (from source) -module.exports.storeItemData = function (item, data, job, done) { +module.exports.storeItemData = function(item, data, job, done) { var log = logger.scopedLog(); - var validate = function (done) { + var validate = function(done) { validateParams([{ name: 'item', variable: item, required: true, requiredProperties: ['user', 'storage', 'save'] }, { @@ -818,20 +818,20 @@ module.exports.storeItemData = function (item, data, job, done) { }], done); }; - var setupLog = function (done) { + var setupLog = function(done) { debug.start('storeItemData'); log = logger.scopedLog({item: item.id}); done(); }; - var updateStorageAttemptedAt = function (done) { + var updateStorageAttemptedAt = function(done) { item.storageAttemptedAt = Date.now(); - item.save(function (error) { + item.save(function(error) { done(error); }); }; // ?? what is going on here?? - var storeFile = function (done) { + var storeFile = function(done) { debug('storeFile : item.user = ', item.user); module.exports.storeFile(item.user, item.storage, item.storagePath, data, (error, storeFileResult) => { if (error) { @@ -847,12 +847,12 @@ module.exports.storeItemData = function (item, data, job, done) { }); }; - var updateStorageProperties = function (storeFileResult, done) { + var updateStorageProperties = function(storeFileResult, done) { item.storageVerifiedAt = Date.now(); item.storageFailedAt = undefined; item.storageBytes = storeFileResult.size; item.storagePath = storeFileResult.path_lower; - item.save(function (error) { + item.save(function(error) { if (!error) { debug.success('updateStorageProperties'); } @@ -861,7 +861,7 @@ module.exports.storeItemData = function (item, data, job, done) { }); }; - var updateJob = function (done) { + var updateJob = function(done) { if (job) { job.incrementTotalItemsStored(); } @@ -869,7 +869,7 @@ module.exports.storeItemData = function (item, data, job, done) { done(); }; - var notifyApp = function (done) { + var notifyApp = function(done) { if (app && typeof app.emit === 'function') { app.emit('storedItemData', item, job); debug('app notified of storedItemData'); @@ -888,13 +888,13 @@ module.exports.storeItemData = function (item, data, job, done) { updateStorageProperties, updateJob, notifyApp - ], function (error) { + ], function(error) { if (error) { log('error', 'Item controller failed to storeItemData', {error: error.message}); if (item && item.save) { item.storageFailedAt = Date.now(); - item.save(function (saveError) { + item.save(function(saveError) { if (saveError) { log('error', 'Item controller failed to update item after failure to store it', {error: saveError.message}); } @@ -919,10 +919,10 @@ module.exports.storeItemData = function (item, data, job, done) { * @param {Object} data - Object that represents data for file. * @param {function} done - Error-first callback function with object representing HTTP response body from storage request as second parameter. */ -module.exports.storeFile = function (user, storage, path, data, done) { +module.exports.storeFile = function(user, storage, path, data, done) { var log = logger.scopedLog(); - var validate = function (done) { + var validate = function(done) { validateParams([{ name: 'user', variable: user, required: true, requiredProperties: ['id'] }, { @@ -933,7 +933,7 @@ module.exports.storeFile = function (user, storage, path, data, done) { name: 'data', variable: data, required: true, requiredType: ['buffer', 'object'] }, { name: 'done', variable: done, required: true, requiredType: 'function' - }], function (error) { + }], function(error) { if (!error) { var mediaType = mime.lookup(path); @@ -950,7 +950,7 @@ module.exports.storeFile = function (user, storage, path, data, done) { }); }; - var prepareData = function (done) { + var prepareData = function(done) { debug.start('storeFile (path: %s)', path); if (!(data instanceof Buffer)) { data = JSON.stringify(data); @@ -959,7 +959,7 @@ module.exports.storeFile = function (user, storage, path, data, done) { done(); }; - var setupLog = function (done) { + var setupLog = function(done) { log = logger.scopedLog({ path: path, storage: storage.id, @@ -969,11 +969,11 @@ module.exports.storeFile = function (user, storage, path, data, done) { done(); }; - var findUserStorageAuth = function (done) { + var findUserStorageAuth = function(done) { UserStorageAuth.findOne({ storage: storage.id, user: user.id - }, function (error, userStorageAuth) { + }, function(error, userStorageAuth) { if (!error && !userStorageAuth) { error = new Error('Failed to retrieve userStorageAuth'); } @@ -982,7 +982,7 @@ module.exports.storeFile = function (user, storage, path, data, done) { }); }; - var storeFile = function (userStorageAuth, done) { + var storeFile = function(userStorageAuth, done) { var options = { body: data, headers: storage.headers(path, userStorageAuth), @@ -992,7 +992,7 @@ module.exports.storeFile = function (user, storage, path, data, done) { debug('storeFile:options %o', options); // what is going on t is ????? - request.post(options, function (error, res, body) { + request.post(options, function(error, res, body) { if (!error) { error = request.statusCodeError(res.statusCode); } @@ -1013,7 +1013,7 @@ module.exports.storeFile = function (user, storage, path, data, done) { setupLog, findUserStorageAuth, storeFile - ], function (error, responseBody) { + ], function(error, responseBody) { if (error) { log('error', 'Item controller failed to store file', {error: error.message, responseBody: responseBody}); } diff --git a/app/lib/jsonapi.js b/app/lib/jsonapi.js index 471ebea..d9a1fd0 100644 --- a/app/lib/jsonapi.js +++ b/app/lib/jsonapi.js @@ -23,7 +23,7 @@ var jsonapi = {}; * @param {Object} document - Mongoose document * @returns {Object} object - JSON API resource object */ -jsonapi.resourceObjectFromDocument = function (document) { +jsonapi.resourceObjectFromDocument = function(document) { if (!document) { throw new Error('No document provided'); } @@ -38,8 +38,8 @@ jsonapi.resourceObjectFromDocument = function (document) { var relationships = {}; - Object.keys(attributes).forEach(function (key) { - var addRelationship = function (property) { + Object.keys(attributes).forEach(function(key) { + var addRelationship = function(property) { if (property && property.id && property.modelType) { if (!relationships[key] || !relationships[key].data) { if (Array.isArray(document[key])) { @@ -100,7 +100,7 @@ jsonapi.resourceObjectFromDocument = function (document) { }; }; -jsonapi.responseDocument = function (data, included, errors) { +jsonapi.responseDocument = function(data, included, errors) { var doc = {}; if (data) { @@ -126,7 +126,7 @@ jsonapi.responseDocument = function (data, included, errors) { * @param {string} method - HTTP method * @returns {string} allowed value (e.g. "public", "user", or "admin) */ -jsonapi.allowed = function (model, method) { +jsonapi.allowed = function(model, method) { if (typeof model.jsonapi[method] === 'string') { return model.jsonapi[method]; } else if (typeof model.jsonapi[method] === 'object') { @@ -158,7 +158,7 @@ jsonapi.compiledQueryConditions = function myself(req, conditions, model, method * @param {string} method - HTTP method (optional) * @param {callback} done */ -jsonapi.modelQueryConditions = function (req, model, method, done) { +jsonapi.modelQueryConditions = function(req, model, method, done) { validateParams([{ name: 'req', variable: req, required: true }, { @@ -195,7 +195,7 @@ jsonapi.modelQueryConditions = function (req, model, method, done) { * @param {Object} relationships - Request relationships * @return {Object} Normalized relationships */ -jsonapi.normalizeRelationships = function (relationships) { +jsonapi.normalizeRelationships = function(relationships) { relationships = Object.assign({}, relationships); // Remove any relationships with empty data properties @@ -218,7 +218,7 @@ jsonapi.normalizeRelationships = function (relationships) { * @param {Object} app - Express app * @param {Object} Model - Mongoose model */ -jsonapi.routeModelDeleteObjectResource = function (app, Model) { +jsonapi.routeModelDeleteObjectResource = function(app, Model) { this.routeModelResource(app, Model, 'delete', '/' + _.kebabCase(Model.modelType()) + '/:id', (req, res) => { var getConditions = (done) => { this.compiledQueryConditions(req, {_id: req.params.id}, Model, 'delete', done); @@ -234,7 +234,7 @@ jsonapi.routeModelDeleteObjectResource = function (app, Model) { } else if (!document) { this.sendNotFound(res); } else { - document.remove(function (error) { + document.remove(function(error) { if (error) { res.status(500).send(); } else { @@ -251,7 +251,7 @@ jsonapi.routeModelDeleteObjectResource = function (app, Model) { * @param {Object} app - Express app * @param {model} model - Mongoose model */ -jsonapi.routeModelGetObjectResource = function (app, Model) { +jsonapi.routeModelGetObjectResource = function(app, Model) { this.routeModelResource(app, Model, 'get', '/' + _.kebabCase(Model.modelType()) + '/:id', (req, res) => { var getConditions = (done) => { this.compiledQueryConditions(req, {_id: req.params.id}, Model, 'get', done); @@ -283,7 +283,7 @@ jsonapi.routeModelGetObjectResource = function (app, Model) { * @param {Object} app - Express app * @param {model} model - Mongoose model */ -jsonapi.routeModelGetObjectsResource = function (app, Model) { +jsonapi.routeModelGetObjectsResource = function(app, Model) { this.routeModelResource(app, Model, 'get', '/' + _.kebabCase(Model.modelType()), (req, res) => { var compileConditions = (done) => { var conditions = {}; @@ -331,7 +331,7 @@ jsonapi.routeModelGetObjectsResource = function (app, Model) { query.exec(done); }; - async.waterfall([compileConditions, executeQuery], function (error, documents) { + async.waterfall([compileConditions, executeQuery], function(error, documents) { if (error) { logger.error('Resource router failed to query for objects', { model: Model.modelName, @@ -350,7 +350,7 @@ jsonapi.routeModelGetObjectsResource = function (app, Model) { * @param {Object} app - Express app * @param {Object} Model - Mongoose model */ -jsonapi.routeModelPatchObjectResource = function (app, Model) { +jsonapi.routeModelPatchObjectResource = function(app, Model) { this.routeModelResource(app, Model, 'patch', '/' + _.kebabCase(Model.modelType()) + '/:id', (req, res) => { var validate = (done) => { this.validateQueryData(req, req.body.data, Model, 'patch', done); @@ -373,7 +373,7 @@ jsonapi.routeModelPatchObjectResource = function (app, Model) { return done(undefined, document); } - this.saveRelationshipsToDocument(document, this.normalizeRelationships(req.body.data.relationships), function (error) { + this.saveRelationshipsToDocument(document, this.normalizeRelationships(req.body.data.relationships), function(error) { done(error, document); }); }; @@ -386,7 +386,7 @@ jsonapi.routeModelPatchObjectResource = function (app, Model) { var executePostRoutine = (document, done) => { if (Model.jsonapi.patch && Model.jsonapi.patch.post) { - Model.jsonapi.patch.post(req, res, document, function (error, req, res, document) { + Model.jsonapi.patch.post(req, res, document, function(error, req, res, document) { done(error, document); }); } else { @@ -425,7 +425,7 @@ jsonapi.routeModelPatchObjectResource = function (app, Model) { * @param {Object} Model - Mongoose model */ -jsonapi.routeModelPostObjectResource = function (app, Model) { +jsonapi.routeModelPostObjectResource = function(app, Model) { debug('routeModelPostObjectResource -- Model = ', Model.collection.collectionName); this.routeModelResource(app, Model, 'post', '/' + _.kebabCase(Model.modelType()), (req, res) => { @@ -460,7 +460,7 @@ jsonapi.routeModelPostObjectResource = function (app, Model) { return done(undefined, document); } - this.saveRelationshipsToDocument(document, this.normalizeRelationships(req.body.data.relationships), function (error) { + this.saveRelationshipsToDocument(document, this.normalizeRelationships(req.body.data.relationships), function(error) { done(error, document); }); }; @@ -491,7 +491,7 @@ jsonapi.routeModelPostObjectResource = function (app, Model) { if (Model.jsonapi.post && Model.jsonapi.post.post) { - Model.jsonapi.post.post(req, res, document, function (error) { + Model.jsonapi.post.post(req, res, document, function(error) { done(error, document); }); } else { @@ -528,7 +528,7 @@ jsonapi.routeModelPostObjectResource = function (app, Model) { * @param {string} path - Path to resource * @param {function} done - Express route callback expecting req and res as parameters */ -jsonapi.routeModelResource = function (app, model, method, path, done) { +jsonapi.routeModelResource = function(app, model, method, path, done) { debug('jsonapi.routeModelResource -- method = %s, path = %s', method, path); if (!model.jsonapi || !model.jsonapi[method]) { return; @@ -554,7 +554,7 @@ jsonapi.routeModelResource = function (app, model, method, path, done) { * @param {Object} document - Mongoose document * @returns {Object} object - JSON API relationship object */ -jsonapi.resourceIdentifierObjectFromDocument = function (document) { +jsonapi.resourceIdentifierObjectFromDocument = function(document) { if (!document) { throw new Error('No document provided'); } @@ -572,7 +572,7 @@ jsonapi.resourceIdentifierObjectFromDocument = function (document) { * @param {string} name - Name of relationship (e.g. "author") * @param {string} [type=to-many] - Type of relationship (either "to-many" or "to-one") */ -jsonapi.addRelationshipToResourceObject = function (object, document, name, type) { +jsonapi.addRelationshipToResourceObject = function(object, document, name, type) { validateParams([{ name: 'object', variable: object, required: true, requiredType: 'object' }, { @@ -616,7 +616,7 @@ jsonapi.addRelationshipToResourceObject = function (object, document, name, type * @param {Object} errors - Errors (optional) * @param {number} [status=200] - HTTP status code */ -jsonapi.sendResponseDocument = function (res, data, included, errors, status) { +jsonapi.sendResponseDocument = function(res, data, included, errors, status) { if (!status) { status = 200; } @@ -630,7 +630,7 @@ jsonapi.sendResponseDocument = function (res, data, included, errors, status) { * @param {Object} data – Principal data * @param {Object} included – Included resources (optional) */ -jsonapi.sendData = function (res, data, included) { +jsonapi.sendData = function(res, data, included) { if (!data) { throw new Error('No data parameter provided'); } @@ -644,7 +644,7 @@ jsonapi.sendData = function (res, data, included) { * @param {Object} document - Model document * @param {number} status - HTTP status code */ -jsonapi.sendDocument = function (res, document, status) { +jsonapi.sendDocument = function(res, document, status) { this.sendResponseDocument(res, jsonapi.resourceObjectFromDocument(document), null, null, status); }; @@ -653,7 +653,7 @@ jsonapi.sendDocument = function (res, document, status) { * @param {Object} res – Express response object * @param {Object} documents - Array of model documents */ -jsonapi.sendDocuments = function (res, documents) { +jsonapi.sendDocuments = function(res, documents) { var objects = documents.map((document) => { return jsonapi.resourceObjectFromDocument(document); }); @@ -665,7 +665,7 @@ jsonapi.sendDocuments = function (res, documents) { * Sends response document with 404 status code * @param {Object} res – Express response object */ -jsonapi.sendNotFound = function (res) { +jsonapi.sendNotFound = function(res) { this.sendResponseDocument(res, null, null, null, 404); }; @@ -675,7 +675,7 @@ jsonapi.sendNotFound = function (res) { * @param {Error} error - Error object (optional) with optional errors property * @param {number} [status=500] - HTTP status code */ -jsonapi.sendError = function (res, error, status) { +jsonapi.sendError = function(res, error, status) { if (error) { var errors = error.errors; @@ -685,12 +685,12 @@ jsonapi.sendError = function (res, error, status) { // Convert object of errors to array if needed if (typeof errors === 'object' && !Array.isArray(errors)) { - errors = Object.keys(errors).map(function (key) { + errors = Object.keys(errors).map(function(key) { return errors[key]; }); } - errors = errors.map(function (error) { + errors = errors.map(function(error) { return { title: error.message }; @@ -707,13 +707,13 @@ jsonapi.sendError = function (res, error, status) { /** * Establish middleware that generates routes conformant to the JSON API specification for app and Mongoose models */ -jsonapi.routeModelResources = function () { +jsonapi.routeModelResources = function() { /** * Negotiate the Content-Type and Accept request headers * @see {@link http://jsonapi.org/format/#content-negotiation-servers} */ - app.use(function (req, res, next) { - var isModifiedContentType = function (contentType) { + app.use(function(req, res, next) { + var isModifiedContentType = function(contentType) { return (/application\/vnd\.api\+json/.test(contentType) && contentType !== 'application/vnd.api+json'); }; @@ -724,7 +724,7 @@ jsonapi.routeModelResources = function () { if (req.get('Accept')) { var badAccept = false; - req.get('Accept').split(';').forEach(function (accept) { + req.get('Accept').split(';').forEach(function(accept) { badAccept = (isModifiedContentType(accept)); }); @@ -737,7 +737,7 @@ jsonapi.routeModelResources = function () { next(); }); - app.use(function (req, res, next) { + app.use(function(req, res, next) { res.set('Content-Type', 'application/vnd.api+json'); next(); }); @@ -785,7 +785,7 @@ jsonapi.routeModelResources = function () { * @param {Object} middleware - Dictionary of middleware boolean or function values to use for route * @param {function} done - Express route callback expecting req and res as parameters */ -jsonapi.routeResource = function (app, method, path, middleware, done) { +jsonapi.routeResource = function(app, method, path, middleware, done) { //debug("jsonapi.routeResource: ",app, method, path, middleware, done); var requireAuthentication = (req, res, next) => { if (middleware && middleware.requireAuthentication) { @@ -822,8 +822,8 @@ jsonapi.routeResource = function (app, method, path, middleware, done) { * @param {Object} document - Key-value object of relationships * @param {callback} done */ -jsonapi.saveRelationshipsToDocument = function (document, relationships, done) { - var validate = function (done) { +jsonapi.saveRelationshipsToDocument = function(document, relationships, done) { + var validate = function(done) { validateParams([{ name: 'document', variable: document, required: true, requiredType: ['object', 'constructor'] }, { @@ -833,11 +833,11 @@ jsonapi.saveRelationshipsToDocument = function (document, relationships, done) { }], done); }; - var saveRelationshipsToDocument = function (done) { + var saveRelationshipsToDocument = function(done) { var Model = models[document.modelId()]; - async.forEachOf(relationships, function (relationship, relationshipName, done) { - var validateRelationship = function (done) { + async.forEachOf(relationships, function(relationship, relationshipName, done) { + var validateRelationship = function(done) { var errors = []; if (!Model.schema.tree[relationshipName]) { @@ -864,9 +864,9 @@ jsonapi.saveRelationshipsToDocument = function (document, relationships, done) { done(error); }; - var addRelationshipToDocument = function (done) { - var validateAndAddResourceIdentifierObjectRelationship = function (resourceObject, done) { - var validateResourceIdentifierObject = function (done) { + var addRelationshipToDocument = function(done) { + var validateAndAddResourceIdentifierObjectRelationship = function(resourceObject, done) { + var validateResourceIdentifierObject = function(done) { var errors = []; if (!resourceObject.id) { @@ -902,7 +902,7 @@ jsonapi.saveRelationshipsToDocument = function (document, relationships, done) { done(error); }; - var addResourceIdentifierToDocument = function (done) { + var addResourceIdentifierToDocument = function(done) { if (Array.isArray(Model.schema.tree[relationshipName]) && document[relationshipName].indexOf(resourceObject.id) === -1) { document[relationshipName].push(resourceObject.id); } else if (!Array.isArray(Model.schema.tree[relationshipName])) { @@ -912,7 +912,7 @@ jsonapi.saveRelationshipsToDocument = function (document, relationships, done) { done(); }; - async.series([validateResourceIdentifierObject, addResourceIdentifierToDocument], function (error) { + async.series([validateResourceIdentifierObject, addResourceIdentifierToDocument], function(error) { done(error, document); }); }; @@ -928,7 +928,7 @@ jsonapi.saveRelationshipsToDocument = function (document, relationships, done) { }, done); }; - async.series([validate, saveRelationshipsToDocument], function (error) { + async.series([validate, saveRelationshipsToDocument], function(error) { done(error, document); }); }; @@ -941,7 +941,7 @@ jsonapi.saveRelationshipsToDocument = function (document, relationships, done) { * @param {string} method - HTTP method (optional) * @param {function} done - Error-first callback function expecting no other parameters */ -jsonapi.validateQueryData = function (req, data, model, method, done) { +jsonapi.validateQueryData = function(req, data, model, method, done) { var getConditions = (done) => { this.modelQueryConditions(req, model, method, done); }; @@ -949,7 +949,7 @@ jsonapi.validateQueryData = function (req, data, model, method, done) { var validateQueryData = (conditions, done) => { var errors = []; - Object.keys(conditions).forEach(function (key) { + Object.keys(conditions).forEach(function(key) { var isEqualObjectId; try { @@ -980,7 +980,7 @@ jsonapi.validateQueryData = function (req, data, model, method, done) { * Returns Express route middleware that validates request body against JSON API specification and URL for optional model * @param {Object} model - Mongoose model (optional) */ -jsonapi.validateRequestBody = function (model) { +jsonapi.validateRequestBody = function(model) { return (req, res, next) => { var errors = []; From 702dac2724f8fd9213229acffa680bcecb02b544 Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Tue, 22 Aug 2017 18:00:35 +0200 Subject: [PATCH 08/14] Further correct ESLint errors and further configure .eslintrc (no-multiple-empty-lines, padded-blocks, padding-line-between-statements, sort-imports, comma-dangle) -- prior to "--fix" --- .eslintrc.js | 16 ++++++++++++++-- .gitignore | 2 -- Gruntfile.js | 2 +- app/controllers/item.js | 6 +----- app/lib/jsonapi.js | 11 +++-------- app/models/source.js | 4 ++-- package.json | 1 - 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 87eae32..9eaac8a 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -58,8 +58,20 @@ module.exports = { "space-before-function-paren": [ "error", "never" + ],"no-multiple-empty-lines": [ + "error", + { "max": 2 } + ],"padded-blocks": [ + "error", + "never" + ],"padding-line-between-statements": [ + "error", + { "blankLine": "always", "prev": "*", "next": "if" }, + ],"sort-imports": [ + "error" + ],"comma-dangle": [ + "error", + "never" ] - - } }; \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9f7168c..9bbece1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,3 @@ .env* data* node_modules -.DS_Store -.idea/** diff --git a/Gruntfile.js b/Gruntfile.js index f6d9021..819b07f 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -43,7 +43,7 @@ module.exports = function(grunt) { }, nodemon: { dev: { - script: 'app/server.js', + script: 'app/server.js' } }, symlink: { diff --git a/app/controllers/item.js b/app/controllers/item.js index a4596de..25235e3 100644 --- a/app/controllers/item.js +++ b/app/controllers/item.js @@ -30,12 +30,8 @@ var urlRegex = require('app/lib/urlRegex'); var UserSourceAuth = require('app/models/userSourceAuth'); var UserStorageAuth = require('app/models/userStorageAuth'); var validateParams = require('app/lib/validateParams'); - - var queue = kue.createQueue(); - - -const STORE_ITEM_DATA = 'storeItemData'; +var STORE_ITEM_DATA = 'storeItemData'; queue.process(STORE_ITEM_DATA, function(queueJob, done) { debug('process queueJob %s', queueJob.id); diff --git a/app/lib/jsonapi.js b/app/lib/jsonapi.js index d9a1fd0..59a21f9 100644 --- a/app/lib/jsonapi.js +++ b/app/lib/jsonapi.js @@ -12,9 +12,6 @@ var logger = require('app/lib/logger'); var models = require('app/models'); var ObjectId = require('mongoose').Types.ObjectId; var validateParams = require('./validateParams'); -const util = require('util'); - - var jsonapi = {}; /** @@ -31,9 +28,7 @@ jsonapi.resourceObjectFromDocument = function(document) { var Model = models[document.modelId()]; var attributes = document.toObject(); - debug('attributes ', util.inspect(attributes)); - - + debug('attributes %O ', attributes); delete attributes.id; var relationships = {}; @@ -43,7 +38,7 @@ jsonapi.resourceObjectFromDocument = function(document) { if (property && property.id && property.modelType) { if (!relationships[key] || !relationships[key].data) { if (Array.isArray(document[key])) { - relationships[key] = {data: []}; + relationships[key] = { data: [] }; } else { relationships[key] = {data: {}}; } @@ -786,7 +781,7 @@ jsonapi.routeModelResources = function() { * @param {function} done - Express route callback expecting req and res as parameters */ jsonapi.routeResource = function(app, method, path, middleware, done) { - //debug("jsonapi.routeResource: ",app, method, path, middleware, done); + var requireAuthentication = (req, res, next) => { if (middleware && middleware.requireAuthentication) { app.requireAuthentication(req, res, next); diff --git a/app/models/source.js b/app/models/source.js index 80bcdd5..f06988e 100644 --- a/app/models/source.js +++ b/app/models/source.js @@ -3,10 +3,9 @@ * @module */ +var templateCompiler = require('es6-template-strings'); var modelFactory = require('app/factories/model'); var nameMethods = require('./methods/name'); -// a way to interporlate strings to get a final URL -var templateCompiler = require('es6-template-strings'); var SourceContentType = require('app/models/sourceContentType'); var methods = Object.assign({ @@ -34,6 +33,7 @@ var methods = Object.assign({ }, /** + * returns the sourceContentTypes associated with this source * @param done {Function} A callback that will be passed a the error (Error) and optionally the sourceContentTypes results */ getSourceContentTypesForSource: function(done) { diff --git a/package.json b/package.json index b9aab53..e99d0a5 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,6 @@ "supertest": "^3.0.0", "underscore": "^1.8.3", "url": "^0.11.0", - "util": "^0.10.3", "winston": "^2.3.0", "winston-mongodb": "^2.0.0" }, From cace2ea7f6389c6c3d89d7d7e4d4f5a3b957bc7f Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Tue, 22 Aug 2017 18:02:59 +0200 Subject: [PATCH 09/14] Further correct ESLint errors and further configure .eslintrc (no-multiple-empty-lines, padded-blocks, padding-line-between-statements, sort-imports, comma-dangle) -- after "--fix" --- Gruntfile.js | 2 +- app/controllers/item.js | 5 ++--- app/lib/jsonapi.js | 3 +-- app/lib/tasks/grunt-repopulate-collections.js | 1 - app/models/contactVerificationRequest.js | 3 +-- app/models/source.js | 3 --- app/models/storage.js | 2 +- fixtures/models.js | 4 ++-- tests/controllers/item/getResource.js | 8 ++++---- tests/controllers/item/persistItemDataObject.js | 4 ++-- 10 files changed, 14 insertions(+), 21 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index 819b07f..12cf24e 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -32,7 +32,7 @@ module.exports = function(grunt) { jsdoc: { build: { options: { - configure : '.jsdoc.json' + configure: '.jsdoc.json' } } }, diff --git a/app/controllers/item.js b/app/controllers/item.js index 25235e3..7860e6d 100644 --- a/app/controllers/item.js +++ b/app/controllers/item.js @@ -408,7 +408,6 @@ module.exports.storeAllForUserStorageSource = function(user, source, storage, jo let getSourceContentTypesFunction = function(done) { - source.getSourceContentTypesForSource(function(err, sourceContentTypes) { if (err) { return done(err); @@ -519,7 +518,6 @@ module.exports.storeAllForUserStorageSourceContentType = function(user, source, * @param {callback} done */ module.exports.storeItemsPage = function(user, source, storage, sourceContentType, pagination, job, done) { - var log = logger.scopedLog(); var ids, page, userSourceAuth; @@ -696,7 +694,7 @@ module.exports.persistItemDataObject = function(itemDataObject, relationships, d storage: relationships.storage.id, source: relationships.source.id, contentType: relationships.sourceContentType.contentType.id, - sourceItem: itemDataObject.id, + sourceItem: itemDataObject.id }; done(); }; @@ -948,6 +946,7 @@ module.exports.storeFile = function(user, storage, path, data, done) { var prepareData = function(done) { debug.start('storeFile (path: %s)', path); + if (!(data instanceof Buffer)) { data = JSON.stringify(data); } diff --git a/app/lib/jsonapi.js b/app/lib/jsonapi.js index 59a21f9..7a14c35 100644 --- a/app/lib/jsonapi.js +++ b/app/lib/jsonapi.js @@ -485,7 +485,6 @@ jsonapi.routeModelPostObjectResource = function(app, Model) { debug('executePostRoutine... ', Model.collection.collectionName); if (Model.jsonapi.post && Model.jsonapi.post.post) { - Model.jsonapi.post.post(req, res, document, function(error) { done(error, document); }); @@ -525,6 +524,7 @@ jsonapi.routeModelPostObjectResource = function(app, Model) { */ jsonapi.routeModelResource = function(app, model, method, path, done) { debug('jsonapi.routeModelResource -- method = %s, path = %s', method, path); + if (!model.jsonapi || !model.jsonapi[method]) { return; } @@ -781,7 +781,6 @@ jsonapi.routeModelResources = function() { * @param {function} done - Express route callback expecting req and res as parameters */ jsonapi.routeResource = function(app, method, path, middleware, done) { - var requireAuthentication = (req, res, next) => { if (middleware && middleware.requireAuthentication) { app.requireAuthentication(req, res, next); diff --git a/app/lib/tasks/grunt-repopulate-collections.js b/app/lib/tasks/grunt-repopulate-collections.js index de22ee1..b80b973 100644 --- a/app/lib/tasks/grunt-repopulate-collections.js +++ b/app/lib/tasks/grunt-repopulate-collections.js @@ -38,7 +38,6 @@ module.exports = function(grunt) { if (!relatedDocument) { done(new Error('Unable to find related document referenced by resourceObject')); } else { - if (toMany) { if (!document[relationshipName]) { document[relationshipName] = []; diff --git a/app/models/contactVerificationRequest.js b/app/models/contactVerificationRequest.js index 53d819e..f72d2de 100644 --- a/app/models/contactVerificationRequest.js +++ b/app/models/contactVerificationRequest.js @@ -95,7 +95,6 @@ module.exports = ContactVerificationRequest = modelFactory.new('ContactVerificat _id: ObjectId(req.body.data.id), code: req.body.data.attributes.code }, (error, contactVerificationRequest) => { - if (!error && !contactVerificationRequest) { error = new Error('No contactVerificationRequest found with id and code'); } @@ -108,7 +107,7 @@ module.exports = ContactVerificationRequest = modelFactory.new('ContactVerificat }); } }); - }, + } }, post: { allowed: 'public', diff --git a/app/models/source.js b/app/models/source.js index f06988e..aceaae5 100644 --- a/app/models/source.js +++ b/app/models/source.js @@ -24,7 +24,6 @@ var methods = Object.assign({ }, itemsGetUrl: function(itemsGetUrlTemplate, $properties) { - if ($properties.next) { return $properties.next; } @@ -41,11 +40,9 @@ var methods = Object.assign({ if (err) { return done(err); } else { - done(err, sourceContentTypes); } }); - } }, nameMethods); diff --git a/app/models/storage.js b/app/models/storage.js index 2cc5309..d6eb6a7 100644 --- a/app/models/storage.js +++ b/app/models/storage.js @@ -69,7 +69,7 @@ module.exports = modelFactory.new('Storage', { */ itemPutUrl: function(path, userStorageAuth) { validateParams([{ - name: 'path', variable: path, required: true, requiredType: 'string', + name: 'path', variable: path, required: true, requiredType: 'string' }, { name: 'userStorageAuth', variable: userStorageAuth, required: true, requiredProperties: ['storageToken'] }]); diff --git a/fixtures/models.js b/fixtures/models.js index b1cd34a..ed8081f 100644 --- a/fixtures/models.js +++ b/fixtures/models.js @@ -167,7 +167,7 @@ module.exports = { name: 'ContentType', type: 'contentTypes', jsonapi: { - delete: 'admin', + delete: 'admin', get: 'public', patch: 'admin', post: 'admin' @@ -406,7 +406,7 @@ module.exports = { sourceToken: String, sourceUser: String, user: { ref: 'User' } - }, + } }, userStorageAuth: { name: 'UserStorageAuth', diff --git a/tests/controllers/item/getResource.js b/tests/controllers/item/getResource.js index da426a6..39247d6 100644 --- a/tests/controllers/item/getResource.js +++ b/tests/controllers/item/getResource.js @@ -11,19 +11,19 @@ describe('itemController.getResource method', function() { assertions.function.callbacks.error(controller.getResource, [{ when: 'no url parameter provided', params: [undefined], - error: 'Parameter url undefined or null', + error: 'Parameter url undefined or null' }, { when: 'url parameter not string', params: [3], - error: 'Parameter url is not a string', + error: 'Parameter url is not a string' }, { when: 'url parameter not a valid URL', params: ['asdf'], - error: 'Parameter url is not a properly formatted string', + error: 'Parameter url is not a properly formatted string' }, { when: 'url parameter has unsupported extension', params: ['http://example.com/foo.xyz'], - error: 'Parameter url indicates unsupported media type', + error: 'Parameter url indicates unsupported media type' }, { when: 'url parameter indicates non-existent resource', params: [wh.jsonUrl], diff --git a/tests/controllers/item/persistItemDataObject.js b/tests/controllers/item/persistItemDataObject.js index 40d5908..d948478 100644 --- a/tests/controllers/item/persistItemDataObject.js +++ b/tests/controllers/item/persistItemDataObject.js @@ -12,11 +12,11 @@ describe('itemController.persistItemDataObject method', function() { assertions.function.callbacks.error(controller.persistItemDataObject, [{ when: 'no itemDataObject parameter provided', params: [undefined, wh.itemRelationships()], - error: 'Parameter itemDataObject undefined or null', + error: 'Parameter itemDataObject undefined or null' }, { when: 'no relationships parameter provided', params: [wh.itemDataObject(), undefined], - error: 'Parameter relationships undefined or null', + error: 'Parameter relationships undefined or null' }, { when: 'itemDataObject parameter has no id property', params: [wh.itemDataObject(), wh.itemRelationships()], From 2c4d9e367ab9bd62f744679661dbd441aa1c8d55 Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Sun, 27 Aug 2017 18:44:57 +0200 Subject: [PATCH 10/14] Add new ESLint rules to put space inside brackets {}; remove debugTools module and factor its code to debug module --- .eslintrc.js | 7 +++-- app/controllers/item.js | 37 +++++++++++------------- app/lib/assertions/it.js | 1 - app/lib/debug.js | 33 +++++++++++++++++++++ app/lib/jsonapi.js | 15 +++++----- app/lib/utils/debuggingTools | 51 --------------------------------- app/models/item.js | 10 +++---- app/models/job.js | 10 +++---- app/models/source.js | 11 ++++--- app/models/sourceContentType.js | 2 +- app/models/userSourceAuth.js | 4 +-- 11 files changed, 79 insertions(+), 102 deletions(-) delete mode 100644 app/lib/utils/debuggingTools diff --git a/.eslintrc.js b/.eslintrc.js index 9eaac8a..0f151f1 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -34,7 +34,7 @@ module.exports = { "always" ], "key-spacing": [ - "warn" + "error" ], "func-call-spacing": [ "error", @@ -60,7 +60,7 @@ module.exports = { "never" ],"no-multiple-empty-lines": [ "error", - { "max": 2 } + { "max": 1 } ],"padded-blocks": [ "error", "never" @@ -72,6 +72,9 @@ module.exports = { ],"comma-dangle": [ "error", "never" + ],"object-curly-spacing": [ + "error", + "always" ] } }; \ No newline at end of file diff --git a/app/controllers/item.js b/app/controllers/item.js index 7860e6d..e5d21ef 100644 --- a/app/controllers/item.js +++ b/app/controllers/item.js @@ -155,7 +155,7 @@ module.exports.getResource = function(url, done) { getResource ], function(error, resource) { if (error) { - log('error', 'Item controller failed to get resource', {error: error}); + log('error', 'Item controller failed to get resource', { error: error }); } done(error, resource); @@ -231,7 +231,6 @@ module.exports.itemsGetUrl = function(source, sourceContentType, userSourceAuth, name: 'pagination', variable: pagination }]); - debug('getItemURL, source.authScope = %s', source.authScope); return source.itemsGetUrl(sourceContentType.itemsGetUrlTemplate, { @@ -275,7 +274,6 @@ module.exports.totalItemsAvailableFromPage = function(page, source, contentType) return total; }; - /** * Returns error from items page if error exists within. * @param {Object} page - Items page. @@ -301,7 +299,6 @@ module.exports.itemsPageError = function(page) { } }; - /** * Returns pagination for next items page after current items page. * @param {Object} page - Current items page. @@ -322,22 +319,22 @@ module.exports.itemsPageNextPagination = function(page, pagination, contentType) if (page.response && page.response[contentType.pluralLowercaseName()] && page.response[contentType.pluralLowercaseName()].items && page.response[contentType.pluralLowercaseName()].items.length) { if (pagination && pagination.offset) { - nextPagination = {offset: pagination.offset + page.response[contentType.pluralLowercaseName()].items.length}; + nextPagination = { offset: pagination.offset + page.response[contentType.pluralLowercaseName()].items.length }; } else { - nextPagination = {offset: page.response[contentType.pluralLowercaseName()].items.length}; + nextPagination = { offset: page.response[contentType.pluralLowercaseName()].items.length }; } } if (page.data && page.data.pagination && page.data.pagination.next_max_id) { - nextPagination = {maxId: page.data.pagination.next_max_id}; + nextPagination = { maxId: page.data.pagination.next_max_id }; } if (page.links && page.links.next) { - nextPagination = {next: page.links.next}; + nextPagination = { next: page.links.next }; } if (page.paging && page.paging.next) { - nextPagination = {next: page.paging.next}; + nextPagination = { next: page.paging.next }; } debug.success('itemsPageNextPagination (nextPagination: %o)', nextPagination); @@ -406,7 +403,6 @@ module.exports.storeAllForUserStorageSource = function(user, source, storage, jo module.exports.storeAllForUserStorageSourceContentType(user, source, storage, sourceContentType, job, done); }; - let getSourceContentTypesFunction = function(done) { source.getSourceContentTypesForSource(function(err, sourceContentTypes) { if (err) { @@ -422,10 +418,9 @@ module.exports.storeAllForUserStorageSource = function(user, source, storage, jo async.eachSeries(sourceContentTypes, storeAllForUserStorageSourceContentType, done); }; - async.waterfall([validate, setupLog, getSourceContentTypesFunction, storeAllItems], function(error) { if (error) { - log('error', 'Item controller failed to store all items', {error: error.message}); + log('error', 'Item controller failed to store all items', { error: error.message }); } else { debug.success('storeAllForUserStorageSource'); } @@ -489,16 +484,16 @@ module.exports.storeAllForUserStorageSourceContentType = function(user, source, } }; - storeAllItemPages(null, {offset: 0}); + storeAllItemPages(null, { offset: 0 }); }; async.series([validate, setupLog, storeAllItems], function(error) { if (error) { debug.error('storeAllForUserStorageSourceContentType (message: %s)', error.message); - log('error', 'Item controller failed to store all items', {error: error}); + log('error', 'Item controller failed to store all items', { error: error }); } else { debug.success('storeAllForUserStorageSourceContentType'); - log('milestone', 'Item controller stored all items', {error: error}); + log('milestone', 'Item controller stored all items', { error: error }); } done(error); @@ -652,7 +647,7 @@ module.exports.storeItemsPage = function(user, source, storage, sourceContentTyp determineNextPagination ], function(error, nextPagination) { if (error) { - log('error', 'Item controller failed to store page of items', {error: error.message}); + log('error', 'Item controller failed to store page of items', { error: error.message }); } else { debug.success('storeItemsPage (sourceContentType: %s, pagination: %o, nextPagination: %o)', sourceContentType.id, pagination, nextPagination); } @@ -781,7 +776,7 @@ module.exports.persistItemDataObject = function(itemDataObject, relationships, d savePath ], function(error, item) { if (error) { - log('error', 'Item controller failed to persist item data object', {error: error}); + log('error', 'Item controller failed to persist item data object', { error: error }); } else { debug.success('persistItemDataObject', item); } @@ -814,7 +809,7 @@ module.exports.storeItemData = function(item, data, job, done) { var setupLog = function(done) { debug.start('storeItemData'); - log = logger.scopedLog({item: item.id}); + log = logger.scopedLog({ item: item.id }); done(); }; @@ -884,13 +879,13 @@ module.exports.storeItemData = function(item, data, job, done) { notifyApp ], function(error) { if (error) { - log('error', 'Item controller failed to storeItemData', {error: error.message}); + log('error', 'Item controller failed to storeItemData', { error: error.message }); if (item && item.save) { item.storageFailedAt = Date.now(); item.save(function(saveError) { if (saveError) { - log('error', 'Item controller failed to update item after failure to store it', {error: saveError.message}); + log('error', 'Item controller failed to update item after failure to store it', { error: saveError.message }); } return done(error); @@ -1010,7 +1005,7 @@ module.exports.storeFile = function(user, storage, path, data, done) { storeFile ], function(error, responseBody) { if (error) { - log('error', 'Item controller failed to store file', {error: error.message, responseBody: responseBody}); + log('error', 'Item controller failed to store file', { error: error.message, responseBody: responseBody }); } done(error, responseBody); diff --git a/app/lib/assertions/it.js b/app/lib/assertions/it.js index e8cbb28..2271f0c 100644 --- a/app/lib/assertions/it.js +++ b/app/lib/assertions/it.js @@ -64,7 +64,6 @@ module.exports = function(description, assertion) { return description; }; - return function(subjectName, subject, tests) { if (!tests) { tests = subject; diff --git a/app/lib/debug.js b/app/lib/debug.js index e2716ff..4e2657e 100644 --- a/app/lib/debug.js +++ b/app/lib/debug.js @@ -54,5 +54,38 @@ module.exports = function(name) { this.style(['gray'], arguments); }; + debug.noAsyncTrace= function(depth=1,compact=false,filter='async.js'){ + this.stackTrace(depth,compact,filter); + }; + + debug.stackTrace = function(depth = 1,compact=false, filter=null) { + depth = depth < 1 ? 1 : depth; + let theCurrentStack = sTrace.get(); + let tmp,fileName,output; + + output = 'trace: '; + depth = depth < theCurrentStack.length -1 ? depth : theCurrentStack.length -1; + + for (let i = 1; i <= depth; i++) { + tmp = theCurrentStack[i]; + fileName = tmp.getFileName(); + + + if (fileName.indexOf('/') > -1) { + fileName = fileName.split('/').pop(); + + if (filter && filter.indexOf(fileName) > -1) continue; + } + + if (compact) { + output += `${fileName}.${tmp.getFunctionName()}, --> `; + } else { + output += `${tmp.getFunctionName()}, ${fileName}, ${tmp.getLineNumber()}\n`; + } + } + + this(output); + }; + return debug; }; \ No newline at end of file diff --git a/app/lib/jsonapi.js b/app/lib/jsonapi.js index 7a14c35..5e9117e 100644 --- a/app/lib/jsonapi.js +++ b/app/lib/jsonapi.js @@ -40,7 +40,7 @@ jsonapi.resourceObjectFromDocument = function(document) { if (Array.isArray(document[key])) { relationships[key] = { data: [] }; } else { - relationships[key] = {data: {}}; + relationships[key] = { data: {} }; } } @@ -86,7 +86,6 @@ jsonapi.resourceObjectFromDocument = function(document) { return _.kebabCase(key); }); - return { id: document.id, type: document.modelType(), @@ -216,7 +215,7 @@ jsonapi.normalizeRelationships = function(relationships) { jsonapi.routeModelDeleteObjectResource = function(app, Model) { this.routeModelResource(app, Model, 'delete', '/' + _.kebabCase(Model.modelType()) + '/:id', (req, res) => { var getConditions = (done) => { - this.compiledQueryConditions(req, {_id: req.params.id}, Model, 'delete', done); + this.compiledQueryConditions(req, { _id: req.params.id }, Model, 'delete', done); }; var findOne = (conditions, done) => { @@ -249,7 +248,7 @@ jsonapi.routeModelDeleteObjectResource = function(app, Model) { jsonapi.routeModelGetObjectResource = function(app, Model) { this.routeModelResource(app, Model, 'get', '/' + _.kebabCase(Model.modelType()) + '/:id', (req, res) => { var getConditions = (done) => { - this.compiledQueryConditions(req, {_id: req.params.id}, Model, 'get', done); + this.compiledQueryConditions(req, { _id: req.params.id }, Model, 'get', done); }; var findOne = (conditions, done) => { @@ -314,7 +313,7 @@ jsonapi.routeModelGetObjectsResource = function(app, Model) { if (Model.jsonapi.sort) { query.sort(Model.jsonapi.sort); } else { - query.sort({createdAt: -1}); + query.sort({ createdAt: -1 }); } } @@ -352,11 +351,11 @@ jsonapi.routeModelPatchObjectResource = function(app, Model) { }; var getConditions = (done) => { - this.compiledQueryConditions(req, {_id: req.params.id}, Model, 'patch', done); + this.compiledQueryConditions(req, { _id: req.params.id }, Model, 'patch', done); }; var findOneAndUpdate = (conditions, done) => { - Model.findOneAndUpdate(conditions, _.camelCasedKeys(req.body.data.attributes), {new: true}, done); + Model.findOneAndUpdate(conditions, _.camelCasedKeys(req.body.data.attributes), { new: true }, done); }; var addRelationships = (document, done) => { @@ -741,7 +740,7 @@ jsonapi.routeModelResources = function() { * Establish body-parser middleware with JSON API error handling */ app.use((req, res, next) => { - var json = bodyParser.json({type: ['application/vnd.api+json', 'application/json']}); + var json = bodyParser.json({ type: ['application/vnd.api+json', 'application/json'] }); json(req, res, (error) => { if (error) { diff --git a/app/lib/utils/debuggingTools b/app/lib/utils/debuggingTools deleted file mode 100644 index 996e971..0000000 --- a/app/lib/utils/debuggingTools +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Various debugging tools - * @module - */ - -let debugTools ={}; - - -let sTrace = require('stack-trace'); -var debug = require('debug')('syncServer:debuggerTools'); - -debugTools.noAsyncTrace= function(depth=1,compact=false,filter='async.js'){ - - this.stackTrace(depth,compact,filter); -}; - -debugTools.stackTrace = function(depth = 1,compact=false, filter=null) { - depth = depth < 1 ? 1 : depth; - let theCurrentStack = sTrace.get(); - let tmp,fileName,output; - - output = 'trace: '; - depth = depth < theCurrentStack.length -1 ? depth : theCurrentStack.length -1; - - for (let i = 1; i <= depth; i++) { - tmp = theCurrentStack[i]; - fileName = tmp.getFileName(); - - - if (fileName.indexOf('/') > -1) { - fileName = fileName.split('/').pop(); - if (filter && filter.indexOf(fileName) > -1) continue; - } - - if (compact) { - output += `${fileName}.${tmp.getFunctionName()}, --> `; - } else { - - - output += `${tmp.getFunctionName()}, ${fileName}, ${tmp.getLineNumber()}\n`; - } - } - - debug(output); - -}; - - - - -module.exports= debugTools; \ No newline at end of file diff --git a/app/models/item.js b/app/models/item.js index 6cce6c8..2e9b941 100644 --- a/app/models/item.js +++ b/app/models/item.js @@ -28,19 +28,19 @@ var convertToFilename = function(content) { * @property {module:models/user~User} user - User for which item was pulled from source */ module.exports = modelFactory.new('Item', { - contentType: {ref: 'ContentType', required: true}, + contentType: { ref: 'ContentType', required: true }, description: String, - source: {ref: 'Source', required: true}, + source: { ref: 'Source', required: true }, sourceCreatedAt: Date, - sourceItem: {type: String, required: true}, - storage: {ref: 'Storage', required: true}, + sourceItem: { type: String, required: true }, + storage: { ref: 'Storage', required: true }, storageAttemptedAt: Date, storageBytes: Number, storageError: String, storageFailedAt: Date, storagePath: String, storageVerifiedAt: Date, - user: {ref: 'User', required: true} + user: { ref: 'User', required: true } }, { jsonapi: { sort: '-storageVerifiedAt', diff --git a/app/models/job.js b/app/models/job.js index fd4d2cc..f5e7ea1 100644 --- a/app/models/job.js +++ b/app/models/job.js @@ -19,7 +19,7 @@ var queryConditions = require('./queryConditions'); * @property {module:models/user~User} [user] - User for which this job should be executed */ module.exports = modelFactory.new('Job', { - contentType: {ref: 'ContentType'}, + contentType: { ref: 'ContentType' }, name: { type: String, required: true, @@ -30,9 +30,9 @@ module.exports = modelFactory.new('Job', { message: '"{VALUE}" is not a supported name value' } }, - source: {ref: 'Source'}, - storage: {ref: 'Storage'}, - user: {ref: 'User'}, + source: { ref: 'Source' }, + storage: { ref: 'Storage' }, + user: { ref: 'User' }, totalItemsAvailable: Number, totalItemsStored: Number }, { @@ -96,7 +96,7 @@ module.exports = modelFactory.new('Job', { }; async.series([populate, runJob], (error) => { - var log = logger.scopedLog({jobId: job.id}); + var log = logger.scopedLog({ jobId: job.id }); if (error) { debug.error('# job failed: %s', error.message); diff --git a/app/models/source.js b/app/models/source.js index aceaae5..a7f1cdb 100644 --- a/app/models/source.js +++ b/app/models/source.js @@ -36,7 +36,7 @@ var methods = Object.assign({ * @param done {Function} A callback that will be passed a the error (Error) and optionally the sourceContentTypes results */ getSourceContentTypesForSource: function(done) { - SourceContentType.find({source: this.id}, function(err, sourceContentTypes) { + SourceContentType.find({ source: this.id }, function(err, sourceContentTypes) { if (err) { return done(err); } else { @@ -67,14 +67,13 @@ module.exports = modelFactory.new('Source', { authScope: Array, clientId: String, clientSecret: String, - itemDataObjectsFromPagePathTemplate: {type: String, default: 'data'}, + itemDataObjectsFromPagePathTemplate: { type: String, default: 'data' }, - - itemStorageEnabled: {type: Boolean, default: false}, + itemStorageEnabled: { type: Boolean, default: false }, host: String, - itemsLimit: {type: Number, default: 25}, + itemsLimit: { type: Number, default: 25 }, logoGlyphPath: String, - name: {type: String, required: true}, + name: { type: String, required: true }, passportStrategy: String, slug: String, totalItemsAvailableFromPagePathTemplate: String diff --git a/app/models/sourceContentType.js b/app/models/sourceContentType.js index 3269513..5901e70 100644 --- a/app/models/sourceContentType.js +++ b/app/models/sourceContentType.js @@ -17,7 +17,7 @@ var nameMethods = require('./methods/name'); module.exports = modelFactory.new('SourceContentType', { source: { ref: 'Source',required: true }, contentType: { ref: 'ContentType',required: true }, - itemsGetUrlTemplate: { type: String, default: 'https://${sourceHost}/${contentTypePluralCamelName}?access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}'} + itemsGetUrlTemplate: { type: String, default: 'https://${sourceHost}/${contentTypePluralCamelName}?access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}' } }, { jsonapi: { diff --git a/app/models/userSourceAuth.js b/app/models/userSourceAuth.js index db601e5..0e2957c 100644 --- a/app/models/userSourceAuth.js +++ b/app/models/userSourceAuth.js @@ -15,10 +15,10 @@ var queryConditions = require('./queryConditions'); * @property {module:models/user~User} [user] - User to authenticate at source */ module.exports = modelFactory.new('UserSourceAuth', { - source: {ref: 'Source', required: true}, + source: { ref: 'Source', required: true }, sourceToken: String, sourceUser: String, - user: {ref: 'User'} + user: { ref: 'User' } }, { jsonapi: { get: { From f9ea3dd8add312481ce2d092d408fcdd84f87279 Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Sun, 27 Aug 2017 18:48:00 +0200 Subject: [PATCH 11/14] Remove .DS_Store files; Remove additions to debug module (from previous commit) --- app/lib/debug.js | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/app/lib/debug.js b/app/lib/debug.js index 4e2657e..e2716ff 100644 --- a/app/lib/debug.js +++ b/app/lib/debug.js @@ -54,38 +54,5 @@ module.exports = function(name) { this.style(['gray'], arguments); }; - debug.noAsyncTrace= function(depth=1,compact=false,filter='async.js'){ - this.stackTrace(depth,compact,filter); - }; - - debug.stackTrace = function(depth = 1,compact=false, filter=null) { - depth = depth < 1 ? 1 : depth; - let theCurrentStack = sTrace.get(); - let tmp,fileName,output; - - output = 'trace: '; - depth = depth < theCurrentStack.length -1 ? depth : theCurrentStack.length -1; - - for (let i = 1; i <= depth; i++) { - tmp = theCurrentStack[i]; - fileName = tmp.getFileName(); - - - if (fileName.indexOf('/') > -1) { - fileName = fileName.split('/').pop(); - - if (filter && filter.indexOf(fileName) > -1) continue; - } - - if (compact) { - output += `${fileName}.${tmp.getFunctionName()}, --> `; - } else { - output += `${tmp.getFunctionName()}, ${fileName}, ${tmp.getLineNumber()}\n`; - } - } - - this(output); - }; - return debug; }; \ No newline at end of file From ce8a544129986b745cc750a60c470e4d8827b992 Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Mon, 28 Aug 2017 14:51:12 +0200 Subject: [PATCH 12/14] Make changes as per code review: various eslint fixes, move getItemURL from source.js to controllers/item.js, add sourceContentType model to fixtures/model.js --- .eslintrc.js | 33 ++++++++++++++++------- app/controllers/item.js | 46 +++++++++++++++++++-------------- app/models/source.js | 18 ++----------- app/models/sourceContentType.js | 9 +++---- fixtures/models.js | 22 ++++++++++++++-- package.json | 2 +- 6 files changed, 75 insertions(+), 55 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 0f151f1..6dbacb0 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -39,17 +39,20 @@ module.exports = { "func-call-spacing": [ "error", "never" - ], "block-spacing": [ + ], + "block-spacing": [ "error" ], "array-bracket-spacing": [ "error" - ], "keyword-spacing": [ + ], + "keyword-spacing": [ "error" ], "semi-spacing": [ "error" - ], "arrow-spacing": [ + ], + "arrow-spacing": [ "error" ], "no-tabs": [ @@ -58,21 +61,31 @@ module.exports = { "space-before-function-paren": [ "error", "never" - ],"no-multiple-empty-lines": [ + ], + "no-multiple-empty-lines": [ "error", { "max": 1 } - ],"padded-blocks": [ + ], + "padded-blocks": [ "error", "never" - ],"padding-line-between-statements": [ + ], + "padding-line-between-statements": [ "error", - { "blankLine": "always", "prev": "*", "next": "if" }, - ],"sort-imports": [ + { + "blankLine": "always", + "prev": "*", + "next": "if" + }, + ], + "sort-imports": [ "error" - ],"comma-dangle": [ + ], + "comma-dangle": [ "error", "never" - ],"object-curly-spacing": [ + ], + "object-curly-spacing": [ "error", "always" ] diff --git a/app/controllers/item.js b/app/controllers/item.js index e5d21ef..c2b626b 100644 --- a/app/controllers/item.js +++ b/app/controllers/item.js @@ -25,15 +25,15 @@ var kue = require('kue'); var logger = require('app/lib/logger'); var mime = require('app/lib/mime'); var request = require('app/lib/request'); +var templateCompiler = require('es6-template-strings'); var Url = require('url'); var urlRegex = require('app/lib/urlRegex'); var UserSourceAuth = require('app/models/userSourceAuth'); var UserStorageAuth = require('app/models/userStorageAuth'); var validateParams = require('app/lib/validateParams'); var queue = kue.createQueue(); -var STORE_ITEM_DATA = 'storeItemData'; -queue.process(STORE_ITEM_DATA, function(queueJob, done) { +queue.process('storeItemData', function(queueJob, done) { debug('process queueJob %s', queueJob.id); var getItem = (done) => { @@ -232,20 +232,26 @@ module.exports.itemsGetUrl = function(source, sourceContentType, userSourceAuth, }]); debug('getItemURL, source.authScope = %s', source.authScope); - - return source.itemsGetUrl(sourceContentType.itemsGetUrlTemplate, { - sourceToken: userSourceAuth.sourceToken, - apiVersion: source.apiVersion, - contentTypePluralCamelName: sourceContentType.contentType.pluralCamelName(), - contentTypePluralLowercaseName: sourceContentType.contentType.pluralLowercaseName(), - sourceHost: source.host, - sourceItemsLimit: source.itemsLimit, - maxId: (typeof pagination !== 'undefined' && pagination.maxId) ? pagination.maxId : undefined, - offset: (typeof pagination !== 'undefined' && pagination.offset) ? pagination.offset : 0, - next: (typeof pagination !== 'undefined' && pagination.next) ? pagination.next : undefined, - sourceName: source.name - - }); + + var property_next = (typeof pagination !== 'undefined' && pagination.next) ? pagination.next : undefined; + + if (property_next) { + return property_next; + } else { + return templateCompiler(sourceContentType.itemsGetUrlTemplate, + { + sourceToken: userSourceAuth.sourceToken, + apiVersion: source.apiVersion, + contentTypePluralCamelName: sourceContentType.contentType.pluralCamelName(), + contentTypePluralLowercaseName: sourceContentType.contentType.pluralLowercaseName(), + sourceHost: source.host, + sourceItemsLimit: source.itemsLimit, + maxId: (typeof pagination !== 'undefined' && pagination.maxId) ? pagination.maxId : undefined, + offset: (typeof pagination !== 'undefined' && pagination.offset) ? pagination.offset : 0, + next: property_next, + sourceName: source.name + }); + } }; /** @@ -403,8 +409,8 @@ module.exports.storeAllForUserStorageSource = function(user, source, storage, jo module.exports.storeAllForUserStorageSourceContentType(user, source, storage, sourceContentType, job, done); }; - let getSourceContentTypesFunction = function(done) { - source.getSourceContentTypesForSource(function(err, sourceContentTypes) { + let getSourceContentTypes = function(done) { + source.getSourceContentTypes(function(err, sourceContentTypes) { if (err) { return done(err); } else { @@ -418,7 +424,7 @@ module.exports.storeAllForUserStorageSource = function(user, source, storage, jo async.eachSeries(sourceContentTypes, storeAllForUserStorageSourceContentType, done); }; - async.waterfall([validate, setupLog, getSourceContentTypesFunction, storeAllItems], function(error) { + async.waterfall([validate, setupLog, getSourceContentTypes, storeAllItems], function(error) { if (error) { log('error', 'Item controller failed to store all items', { error: error.message }); } else { @@ -620,7 +626,7 @@ module.exports.storeItemsPage = function(user, source, storage, sourceContentTyp } // this is where the actual jobs queue "items" are created /// where the magic happens… - var queueJob = queue.create(STORE_ITEM_DATA, jobAttributes).save((error) => { + var queueJob = queue.create('storeItemData', jobAttributes).save((error) => { if (error) { debug.error('queueJob %s failed to queue: %s', queueJob.id, error.message); } else { diff --git a/app/models/source.js b/app/models/source.js index a7f1cdb..bb46c75 100644 --- a/app/models/source.js +++ b/app/models/source.js @@ -23,26 +23,14 @@ var methods = Object.assign({ }); }, - itemsGetUrl: function(itemsGetUrlTemplate, $properties) { - if ($properties.next) { - return $properties.next; - } - return templateCompiler(itemsGetUrlTemplate, $properties); - }, /** * returns the sourceContentTypes associated with this source * @param done {Function} A callback that will be passed a the error (Error) and optionally the sourceContentTypes results */ - getSourceContentTypesForSource: function(done) { - SourceContentType.find({ source: this.id }, function(err, sourceContentTypes) { - if (err) { - return done(err); - } else { - done(err, sourceContentTypes); - } - }); + getSourceContentTypes: function(done) { + SourceContentType.find({ source: this.id }, done); } }, nameMethods); @@ -58,7 +46,6 @@ var methods = Object.assign({ * @property {string} [logoGlyphPath] - URL path to logo glyph image file on host (e.g. "/images/logos/foursquare-glyph.svg") * @property {string} name - Name of source (e.g. "foursquare") * @property {string} [passportStrategy] - Strategy for Passport module (e.g. "passport-foursquare") - * @property {string} [itemsGetUrlTemplate=https://${host}/${contentTypePluralCamelName}?access_token=${accessToken}&limit=${limit}&offset=${offset}] - String template used to generate URLs for GET requests for items on source * @property {string} [itemDataObjectsFromPagePathTemplate=data] - String template used to generate object paths to itemDataObjects found within pages returned from source * @property {string} [totalItemsAvailableFromPagePathTemplate=response.${contentTypePluralCamelName}.count] - String template used to generate object paths to value representing total items available for contentType within pages returned from source */ @@ -68,7 +55,6 @@ module.exports = modelFactory.new('Source', { clientId: String, clientSecret: String, itemDataObjectsFromPagePathTemplate: { type: String, default: 'data' }, - itemStorageEnabled: { type: Boolean, default: false }, host: String, itemsLimit: { type: Number, default: 25 }, diff --git a/app/models/sourceContentType.js b/app/models/sourceContentType.js index 5901e70..8108572 100644 --- a/app/models/sourceContentType.js +++ b/app/models/sourceContentType.js @@ -15,14 +15,11 @@ var nameMethods = require('./methods/name'); * @property {module:models/contentType~ContentType} contentType for this sourceContentType */ module.exports = modelFactory.new('SourceContentType', { - source: { ref: 'Source',required: true }, contentType: { ref: 'ContentType',required: true }, - itemsGetUrlTemplate: { type: String, default: 'https://${sourceHost}/${contentTypePluralCamelName}?access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}' } - + itemsGetUrlTemplate: { type: String, default: 'https://${sourceHost}/${contentTypePluralCamelName}?access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}' }, + source: { ref: 'Source',required: true } }, { jsonapi: { - get: 'public', - post: 'public', - del: 'public' + get: 'public' } }, nameMethods); \ No newline at end of file diff --git a/fixtures/models.js b/fixtures/models.js index ed8081f..bdb3763 100644 --- a/fixtures/models.js +++ b/fixtures/models.js @@ -301,7 +301,6 @@ module.exports = { itemStorageEnabled: true, host: 'sourcehost.example.com', itemDataObjectsFromPagePathTemplate: 'things.${contentTypePluralCamelName}', - itemsGetUrlTemplate: 'https://${host}/test-path/${contentTypePluralCamelName}?foo=bar&access_token=${accessToken}&limit=${limit}&offset=${offset}', itemsLimit: 72, logoGlyphPath: '/source/logoGlyphPath.svg', name: 'Super Source', @@ -320,12 +319,31 @@ module.exports = { logoGlyphPath: String, name: { type: String, required: true }, passportStrategy: String, - itemsGetUrlTemplate: String, itemDataObjectsFromPagePathTemplate: { type: String, default: 'data' }, slug: String, totalItemsAvailableFromPagePathTemplate: String } }, + sourceContentType: { + name: 'SourceContentType', + type: 'sourceContentTypes', + jsonapi: { + get: 'public' + }, + mockProperties: () => { + return { + _id: ObjectId(), + contentType: ObjectId(), + itemsGetUrlTemplate: 'https://${sourceHost}/foo/${contentTypePluralCamelName}?bar=wiz&access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}', + source: ObjectId() + }; + }, + schemaProperties: { + contentType: { ref: 'ContentType', required: true }, + itemsGetUrlTemplate: { type: String, default: 'https://${sourceHost}/${contentTypePluralCamelName}?access_token=${sourceToken}&limit=${sourceItemsLimit}&offset=${offset}' }, + source: { ref: 'Source', required: true } + } + }, storage: { name: 'Storage', type: 'storages', diff --git a/package.json b/package.json index e99d0a5..c5458cd 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "debug": "^2.6.0", "emoji-strip": "^1.0.0", "es6-template-strings": "^2.0.1", - "eslint": "^3.17.1", + "eslint": "^4.5.0", "express": "~4.15.0", "express-session": "^1.14.1", "flat": "^2.0.1", From 44a89c6d471ef02c961355ffae31527d2793b0c2 Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Sat, 16 Sep 2017 21:08:08 +0200 Subject: [PATCH 13/14] Correct errors resulting from refactoring code to reflect changing references to sourceContentType to contentType --- app/controllers/item.js | 124 +++++++++++++++++++++------------------- app/models/source.js | 22 +++++-- 2 files changed, 82 insertions(+), 64 deletions(-) diff --git a/app/controllers/item.js b/app/controllers/item.js index c2b626b..440cde1 100644 --- a/app/controllers/item.js +++ b/app/controllers/item.js @@ -25,6 +25,7 @@ var kue = require('kue'); var logger = require('app/lib/logger'); var mime = require('app/lib/mime'); var request = require('app/lib/request'); +var SourceContentType = require('app/models/sourceContentType'); var templateCompiler = require('es6-template-strings'); var Url = require('url'); var urlRegex = require('app/lib/urlRegex'); @@ -212,19 +213,16 @@ module.exports.itemDataObjectsFromPage = function(page, source, contentType) { /** * Return URL for making a GET request for items from source. * @param {Object} source - Source from which to retrieve items. - * @param {Object} sourceContentType - sourceContentType of items. + * @param {Object} contentType - contentType of items. * @param {Object} userSourceAuth - UserSourceAuth used to make request. * @param {Object} pagination - Pagination used to make request. * @returns {string} URL for making a GET request */ -module.exports.itemsGetUrl = function(source, sourceContentType, userSourceAuth, pagination) { +module.exports.itemsGetUrl = function(source, contentType, userSourceAuth, pagination,done) { validateParams([{ name: 'source', variable: source, required: true, requiredProperties: ['host'] }, { - name: 'sourceContentType', - variable: sourceContentType, - required: true, - requiredProperties: ['contentType', 'itemsGetUrlTemplate'] + name: 'contentType', variable: contentType, required: true, requiredProperties: ['name'] }, { name: 'userSourceAuth', variable: userSourceAuth, required: true, requiredProperties: ['sourceToken'] }, { @@ -236,21 +234,32 @@ module.exports.itemsGetUrl = function(source, sourceContentType, userSourceAuth, var property_next = (typeof pagination !== 'undefined' && pagination.next) ? pagination.next : undefined; if (property_next) { - return property_next; + return done(undefined, property_next); } else { - return templateCompiler(sourceContentType.itemsGetUrlTemplate, - { - sourceToken: userSourceAuth.sourceToken, - apiVersion: source.apiVersion, - contentTypePluralCamelName: sourceContentType.contentType.pluralCamelName(), - contentTypePluralLowercaseName: sourceContentType.contentType.pluralLowercaseName(), - sourceHost: source.host, - sourceItemsLimit: source.itemsLimit, - maxId: (typeof pagination !== 'undefined' && pagination.maxId) ? pagination.maxId : undefined, - offset: (typeof pagination !== 'undefined' && pagination.offset) ? pagination.offset : 0, - next: property_next, - sourceName: source.name - }); + SourceContentType.findOne({ + contentType: contentType.id, + source: source.id + }, function(error, sourceContentType) { + console.log("made it here",sourceContentType.itemsGetUrlTemplate); + if (error) { + return done(error); + } else { + var urlToReturn = templateCompiler(sourceContentType.itemsGetUrlTemplate, + { + sourceToken: userSourceAuth.sourceToken, + apiVersion: source.apiVersion, + contentTypePluralCamelName: sourceContentType.contentType.pluralCamelName(), + contentTypePluralLowercaseName: sourceContentType.contentType.pluralLowercaseName(), + sourceHost: source.host, + sourceItemsLimit: source.itemsLimit, + maxId: (typeof pagination !== 'undefined' && pagination.maxId) ? pagination.maxId : undefined, + offset: (typeof pagination !== 'undefined' && pagination.offset) ? pagination.offset : 0, + next: property_next, + sourceName: source.name + }); + return done(undefined, urlToReturn); + } + }); } }; @@ -405,26 +414,20 @@ module.exports.storeAllForUserStorageSource = function(user, source, storage, jo done(); }; - var storeAllForUserStorageSourceContentType = function(sourceContentType, done) { - module.exports.storeAllForUserStorageSourceContentType(user, source, storage, sourceContentType, job, done); + var storeAllForUserStorageContentType = function(contentType, done) { + module.exports.storeAllForUserStorageContentType(user, source, storage, contentType, job, done); }; - let getSourceContentTypes = function(done) { - source.getSourceContentTypes(function(err, sourceContentTypes) { - if (err) { - return done(err); - } else { - done(err, sourceContentTypes); - } - }); + var getContentTypes = function(done) { + source.getContentTypes(done); }; - var storeAllItems = function(sourceContentTypes, done) { - debug.start('storeAllItems (sourceContentTypes: %s)', sourceContentTypes.length); - async.eachSeries(sourceContentTypes, storeAllForUserStorageSourceContentType, done); + var storeAllItems = function(contentTypes, done) { + debug.start('storeAllItems (contentTypes: %s)', contentTypes.length); + async.eachSeries(contentTypes, storeAllForUserStorageContentType, done); }; - async.waterfall([validate, setupLog, getSourceContentTypes, storeAllItems], function(error) { + async.waterfall([validate, setupLog, getContentTypes, storeAllItems], function(error) { if (error) { log('error', 'Item controller failed to store all items', { error: error.message }); } else { @@ -442,12 +445,12 @@ module.exports.storeAllForUserStorageSource = function(user, source, storage, jo * @param {User} user - User for which to retrieve items from source and store them in storage. * @param {Source} source - Source from which to retrieve items. * @param {Storage} storage - Storage within which to store items. - * @param {SourceContentType} sourceContentType - sourceContentType of which to retrieve items. + * @param {ContentType} contentType - contentType of which to retrieve items. * @param {Job} [job] - Job for which to store items. * @param {callback} done */ -module.exports.storeAllForUserStorageSourceContentType = function(user, source, storage, sourceContentType, job, done) { - debug('storeAllForUserStorageSourceContentType, source = %s, storage = %s, sourceContentType = %s', source, storage, sourceContentType); +module.exports.storeAllForUserStorageContentType = function(user, source, storage, contentType, job, done) { + debug('storeAllForUserStorageContentType, source = %s, storage = %s, contentType = %s', source, storage, contentType); var log = logger.scopedLog(); var validate = function(done) { @@ -458,18 +461,18 @@ module.exports.storeAllForUserStorageSourceContentType = function(user, source, }, { name: 'storage', variable: storage, required: true, requiredProperties: ['id'] }, { - name: 'sourceContentType', variable: sourceContentType, required: true, requiredProperties: ['id'] + name: 'contentType', variable: contentType, required: true, requiredProperties: ['id'] }], done); }; var setupLog = function(done) { - debug.start('storeAllForUserStorageSourceContentType'); + debug.start('storeAllForUserStorageContentType'); log = logger.scopedLog({ user: user.id, source: source.id, storage: storage.id, - sourceContentType: sourceContentType.id + contentType: contentType.id }); done(); @@ -483,7 +486,7 @@ module.exports.storeAllForUserStorageSourceContentType = function(user, source, } } else { if (pagination) { - module.exports.storeItemsPage(user, source, storage, sourceContentType, pagination, job, myself); + module.exports.storeItemsPage(user, source, storage, contentType, pagination, job, myself); } else if (done) { done(); } @@ -495,10 +498,10 @@ module.exports.storeAllForUserStorageSourceContentType = function(user, source, async.series([validate, setupLog, storeAllItems], function(error) { if (error) { - debug.error('storeAllForUserStorageSourceContentType (message: %s)', error.message); + debug.error('storeAllForUserStorageContentType (message: %s)', error.message); log('error', 'Item controller failed to store all items', { error: error }); } else { - debug.success('storeAllForUserStorageSourceContentType'); + debug.success('storeAllForUserStorageContentType'); log('milestone', 'Item controller stored all items', { error: error }); } @@ -513,12 +516,12 @@ module.exports.storeAllForUserStorageSourceContentType = function(user, source, * @param {User} user - User for which to retrieve items from source and store them in storage. * @param {Source} source - Source from which to retrieve items. * @param {Storage} storage - Storage within which to store items. - * @param {SourceContentType} sourceContentType - SourceContentType of which to retrieve items. + * @param {ContentType} contentType - contentType of which to retrieve items. * @param {Object} pagination – Object containing pagination information. * @param {Job} [job] - Job for which to store items. * @param {callback} done */ -module.exports.storeItemsPage = function(user, source, storage, sourceContentType, pagination, job, done) { +module.exports.storeItemsPage = function(user, source, storage, contentType, pagination, job, done) { var log = logger.scopedLog(); var ids, page, userSourceAuth; @@ -530,20 +533,20 @@ module.exports.storeItemsPage = function(user, source, storage, sourceContentTyp }, { name: 'storage', variable: storage, required: true, requiredProperties: ['id'] }, { - name: 'sourceContentType', variable: sourceContentType, required: true, requiredProperties: ['id'] + name: 'contentType', variable: contentType, required: true, requiredProperties: ['id'] }, { name: 'pagination', variable: pagination, required: true }], done); }; var setupLog = function(done) { - debug.start('## storeItemsPage (sourceContentType: %s, pagination: %o)', sourceContentType.id, pagination); + debug.start('## storeItemsPage (contentType: %s, pagination: %o)', contentType.id, pagination); ids = { user: user.id, storage: storage.id, source: source.id, - sourceContentType: sourceContentType.id + contentType: contentType.id }; log = logger.scopedLog(Object.assign({}, pagination, ids)); @@ -567,8 +570,12 @@ module.exports.storeItemsPage = function(user, source, storage, sourceContentTyp // get the page of date from URL // whcih will be converted INTO items - var getItemsPageResource = function(done) { - module.exports.getResource(module.exports.itemsGetUrl(source, sourceContentType, userSourceAuth, pagination), done); + var getItemsUrlFromTemplate = function(done) { + module.exports.itemsGetUrl(source, contentType, userSourceAuth, pagination,done); + }; + + var getItemsPageResource = function(url, done) { + module.exports.getResource(url, done); }; var getItemDataObjects = function(resource, done) { @@ -579,15 +586,15 @@ module.exports.storeItemsPage = function(user, source, storage, sourceContentTyp return done(new Error('Failed to retrieve valid item objects page. ' + error.message)); } - var itemDataObjects = module.exports.itemDataObjectsFromPage(page, source, sourceContentType.contentType); - var totalItemsAvailable = module.exports.totalItemsAvailableFromPage(page, source, sourceContentType.contentType); + var itemDataObjects = module.exports.itemDataObjectsFromPage(page, source, contentType); + var totalItemsAvailable = module.exports.totalItemsAvailableFromPage(page, source, contentType); if (job && totalItemsAvailable && pagination.offset === 0) { job.updateTotalItemsAvailable(totalItemsAvailable); } if (!itemDataObjects || !itemDataObjects.length) { - debug.warning('storeItemsPage retrieved page with no data (sourceContentType: %s, pagination: %o)', sourceContentType.id, pagination); + debug.warning('storeItemsPage retrieved page with no data (contentType: %s, pagination: %o)', contentType.id, pagination); } done(undefined, itemDataObjects); @@ -604,7 +611,7 @@ module.exports.storeItemsPage = function(user, source, storage, sourceContentTyp user: user, storage: storage, source: source, - sourceContentType: sourceContentType + contentType: contentType }, (error, item) => { done(error, { item: item, @@ -639,13 +646,14 @@ module.exports.storeItemsPage = function(user, source, storage, sourceContentTyp }; var determineNextPagination = function(done) { - done(undefined, module.exports.itemsPageNextPagination(page, pagination, sourceContentType.contentType)); + done(undefined, module.exports.itemsPageNextPagination(page, pagination, contentType)); }; async.waterfall([ validate, setupLog, findUserSourceAuth, + getItemsUrlFromTemplate, getItemsPageResource, getItemDataObjects, persistItemDataObjects, @@ -655,7 +663,7 @@ module.exports.storeItemsPage = function(user, source, storage, sourceContentTyp if (error) { log('error', 'Item controller failed to store page of items', { error: error.message }); } else { - debug.success('storeItemsPage (sourceContentType: %s, pagination: %o, nextPagination: %o)', sourceContentType.id, pagination, nextPagination); + debug.success('storeItemsPage (contentType: %s, pagination: %o, nextPagination: %o)', contentType.id, pagination, nextPagination); } done(error, nextPagination); @@ -683,7 +691,7 @@ module.exports.persistItemDataObject = function(itemDataObject, relationships, d name: 'relationships', variable: relationships, required: true, - requiredProperties: ['user', 'storage', 'source', 'sourceContentType'] + requiredProperties: ['user', 'storage', 'source', 'contentType'] }], done); }; @@ -694,7 +702,7 @@ module.exports.persistItemDataObject = function(itemDataObject, relationships, d user: relationships.user.id, storage: relationships.storage.id, source: relationships.source.id, - contentType: relationships.sourceContentType.contentType.id, + contentType: relationships.contentType.id, sourceItem: itemDataObject.id }; done(); diff --git a/app/models/source.js b/app/models/source.js index bb46c75..5ccbaca 100644 --- a/app/models/source.js +++ b/app/models/source.js @@ -3,6 +3,7 @@ * @module */ +var debug = require('app/lib/debug')('syncServer:models/source.js'); var templateCompiler = require('es6-template-strings'); var modelFactory = require('app/factories/model'); var nameMethods = require('./methods/name'); @@ -23,15 +24,24 @@ var methods = Object.assign({ }); }, - - /** - * returns the sourceContentTypes associated with this source - * @param done {Function} A callback that will be passed a the error (Error) and optionally the sourceContentTypes results + * returns the contentTypes associated with this source + * @param done {Function} A callback that will be passed a the error (Error) and optionally the contentTypes results */ - getSourceContentTypes: function(done) { - SourceContentType.find({ source: this.id }, done); + getContentTypes: function(done) { + debug(this); + debug("this.id = %O",this.id); + + SourceContentType.find({ source: this._id }, function(err, sourceContentTypes) { + if (err) { + return done(err); + } else { + debug("sourceContentTypes = %O", sourceContentTypes); + done(err, sourceContentTypes.map((sourceContentType) => sourceContentType.contentType)); + } + }); } + }, nameMethods); /** From 0b9c6c98bba411f074866aa27d2d6f4caafd2dab Mon Sep 17 00:00:00 2001 From: pulpopurpura Date: Sun, 8 Oct 2017 14:10:12 +0200 Subject: [PATCH 14/14] Fix ESLint Test Errors --- .eslintrc.js | 2 +- app/controllers/item.js | 1 - app/lib/mailer.js | 1 + app/lib/urlRegex.js | 1 + app/models/source.js | 4 ++-- package.json | 3 ++- 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 6dbacb0..8afa3f5 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -76,7 +76,7 @@ module.exports = { "blankLine": "always", "prev": "*", "next": "if" - }, + } ], "sort-imports": [ "error" diff --git a/app/controllers/item.js b/app/controllers/item.js index 440cde1..435d937 100644 --- a/app/controllers/item.js +++ b/app/controllers/item.js @@ -240,7 +240,6 @@ module.exports.itemsGetUrl = function(source, contentType, userSourceAuth, pagin contentType: contentType.id, source: source.id }, function(error, sourceContentType) { - console.log("made it here",sourceContentType.itemsGetUrlTemplate); if (error) { return done(error); } else { diff --git a/app/lib/mailer.js b/app/lib/mailer.js index 4f02d36..db741cf 100644 --- a/app/lib/mailer.js +++ b/app/lib/mailer.js @@ -5,6 +5,7 @@ var sendGridTransport = require('nodemailer-sendgrid-transport'); var stubTransport = require('nodemailer-stub-transport'); var mailer = { + // eslint-disable-next-line no-useless-escape emailRegex: /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/ }; diff --git a/app/lib/urlRegex.js b/app/lib/urlRegex.js index 7637e59..141cf7f 100644 --- a/app/lib/urlRegex.js +++ b/app/lib/urlRegex.js @@ -1,4 +1,5 @@ /** * Regex to validate URLs */ +// eslint-disable-next-line no-useless-escape module.exports = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%_\+.~#?&//=]{2,256}\.[a-z]{2,4}\b(\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?/; \ No newline at end of file diff --git a/app/models/source.js b/app/models/source.js index 5ccbaca..fd4cf2c 100644 --- a/app/models/source.js +++ b/app/models/source.js @@ -30,13 +30,13 @@ var methods = Object.assign({ */ getContentTypes: function(done) { debug(this); - debug("this.id = %O",this.id); + debug('this.id = %O',this.id); SourceContentType.find({ source: this._id }, function(err, sourceContentTypes) { if (err) { return done(err); } else { - debug("sourceContentTypes = %O", sourceContentTypes); + debug('sourceContentTypes = %O', sourceContentTypes); done(err, sourceContentTypes.map((sourceContentType) => sourceContentType.contentType)); } }); diff --git a/package.json b/package.json index c5458cd..51c1603 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "grunt-contrib-clean": "^1.0.0", "grunt-contrib-symlink": "^1.0.0", "grunt-contrib-watch": "^1.0.0", - "grunt-eslint": "^19.0.0", + "grunt-eslint": "^20.1.0", "grunt-force-task": "^2.0.0", "grunt-jsdoc": "^2.1.0", "grunt-mocha-test": "^0.13.2", @@ -86,6 +86,7 @@ "devDependencies": { "grunt-deploy-files": "1.0.5", "grunt-env": "^0.4.4", + "grunt-eslint": "^19.0.0", "grunt-hoist": "1.0.4" } }