diff --git a/.eslintrc b/.eslintrc index ab4fb41..8976c15 100644 --- a/.eslintrc +++ b/.eslintrc @@ -51,7 +51,6 @@ "no-trailing-spaces": 2, "no-underscore-dangle": 0, "no-unneeded-ternary": 1, - "one-var": 2, "quotes": [2, "single", "avoid-escape"], "semi": [2, "always"], "keyword-spacing": 2, diff --git a/postgres/client.js b/postgres/client.js index b400fca..4f535be 100644 --- a/postgres/client.js +++ b/postgres/client.js @@ -17,7 +17,8 @@ const { { isList, isUri } = require('clayutils'), TransformStream = require('../services/list-transform-stream'), META_PUT_PATCH_FN = patch('meta'); -var knex, log = require('../services/log').setup({ file: __filename }); +var knex, + log = require('../services/log').setup({ file: __filename }); /** * Connect to the default DB and create the Clay @@ -241,24 +242,33 @@ function batch(ops) { } /** - * Return a readable stream of query results - * from the db - * - * @param {Object} options - * @return {Stream} + * Gets a list of components as a readable stream, can handle pagination. + * @param {Object} options + * @returns {Stream} */ function createReadStream(options) { - const { prefix, values, keys } = options, - transform = TransformStream(options), - selects = []; + const { prefix, values, keys, previous, size } = options; + const transform = TransformStream(options); + const selects = []; if (keys) selects.push('id'); if (values) selects.push('data'); - baseQuery(prefix) - .select(...selects) - .where('id', 'like', `${prefix}%`) - .pipe(transform); + const query = baseQuery(prefix); + + query.select(...selects); + query.where('id', 'like', `${prefix}%`); + + if (previous) { + query.where('id', '>', previous); + } + + if (size) { + query.limit(size); + query.orderBy('id', 'asc'); + } + + query.pipe(transform); return transform; } diff --git a/postgres/client.test.js b/postgres/client.test.js index 0ce693b..1d222bb 100644 --- a/postgres/client.test.js +++ b/postgres/client.test.js @@ -435,12 +435,18 @@ describe('postgres/client', () => { describe('createReadStream', () => { const pipe = jest.fn(() => ({})), - where = jest.fn(() => ({ pipe })), - select = jest.fn(() => ({ where })), - withSchema = jest.fn(() => ({ select })), + where = jest.fn(() => ({})), + select = jest.fn(() => ({})), + withSchema = jest.fn(() => ({})), + limit = jest.fn(() => ({})), + orderBy = jest.fn(() => ({})), knex = jest.fn(() => ({ withSchema, - select + select, + where, + limit, + orderBy, + pipe, })), mockedTransform = {}; @@ -454,7 +460,7 @@ describe('postgres/client', () => { const options = { prefix: 'nymag.com/_uris', values: true, - keys: true + keys: true, }, transform = client.createReadStream(options); @@ -474,7 +480,7 @@ describe('postgres/client', () => { const options = { prefix: 'nymag.com/_uris', values: false, - keys: false + keys: false, }, transform = client.createReadStream(options); @@ -486,6 +492,39 @@ describe('postgres/client', () => { expect(where.mock.calls[0][2]).toBe(`${options.prefix}%`); expect(transform).toBe(mockedTransform); }); + + test('queries with a limit and order when page size is set', () => { + TransformStream.mockReturnValueOnce(mockedTransform); + + const options = { + prefix: 'nymag.com/_uris', + values: false, + keys: false, + size: 20, + }; + + client.createReadStream(options); + + expect(limit.mock.calls[0][0]).toBe(20); + expect(orderBy.mock.calls.length).toBe(1); + expect(orderBy.mock.calls[0]).toEqual(['id', 'asc']); + }); + + test('queries with where id > previous when previous is set', () => { + const options = { + prefix: 'nymag.com/_uris', + values: false, + keys: false, + size: 20, + previous: 'nymag.com/components/ad/instances/aaa', + }; + + client.createReadStream(options); + + expect(where.mock.calls[1]).toEqual(['id', '>', options.previous]); + expect(orderBy.mock.calls.length).toBe(1); + expect(orderBy.mock.calls[0]).toEqual(['id', 'asc']); + }); }); describe('put', () => {