From 13c9925d3a5bbe2ab06e9ad058004c8f63924ad2 Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Mon, 27 May 2019 11:35:00 -0400 Subject: [PATCH 01/14] Adds sql queries to migrate pages, users, uris and lists tables adding created_at and updated_at columns in the schema --- .../migrations/005_add_created_at_uris_table.sql | 16 ++++++++++++++++ .../006_add_created_at_lists_table.sql | 16 ++++++++++++++++ .../007_add_created_at_pages_table.sql | 16 ++++++++++++++++ .../008_add_created_at_users_table.sql | 16 ++++++++++++++++ 4 files changed, 64 insertions(+) create mode 100644 services/migrations/005_add_created_at_uris_table.sql create mode 100644 services/migrations/006_add_created_at_lists_table.sql create mode 100644 services/migrations/007_add_created_at_pages_table.sql create mode 100644 services/migrations/008_add_created_at_users_table.sql diff --git a/services/migrations/005_add_created_at_uris_table.sql b/services/migrations/005_add_created_at_uris_table.sql new file mode 100644 index 0000000..3cfbf28 --- /dev/null +++ b/services/migrations/005_add_created_at_uris_table.sql @@ -0,0 +1,16 @@ +CREATE OR REPLACE FUNCTION trigger_set_timestamp() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +ALTER TABLE IF EXISTS uris +ADD COLUMN created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), +ADD COLUMN updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); + +CREATE TRIGGER set_timestamp +BEFORE UPDATE ON uris +FOR EACH ROW +EXECUTE PROCEDURE trigger_set_timestamp(); diff --git a/services/migrations/006_add_created_at_lists_table.sql b/services/migrations/006_add_created_at_lists_table.sql new file mode 100644 index 0000000..e450f6d --- /dev/null +++ b/services/migrations/006_add_created_at_lists_table.sql @@ -0,0 +1,16 @@ +CREATE OR REPLACE FUNCTION trigger_set_timestamp() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +ALTER TABLE IF EXISTS lists +ADD COLUMN created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), +ADD COLUMN updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); + +CREATE TRIGGER set_timestamp +BEFORE UPDATE ON uris +FOR EACH ROW +EXECUTE PROCEDURE trigger_set_timestamp(); diff --git a/services/migrations/007_add_created_at_pages_table.sql b/services/migrations/007_add_created_at_pages_table.sql new file mode 100644 index 0000000..f2e496f --- /dev/null +++ b/services/migrations/007_add_created_at_pages_table.sql @@ -0,0 +1,16 @@ +CREATE OR REPLACE FUNCTION trigger_set_timestamp() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +ALTER TABLE IF EXISTS pages +ADD COLUMN created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), +ADD COLUMN updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); + +CREATE TRIGGER set_timestamp +BEFORE UPDATE ON uris +FOR EACH ROW +EXECUTE PROCEDURE trigger_set_timestamp(); diff --git a/services/migrations/008_add_created_at_users_table.sql b/services/migrations/008_add_created_at_users_table.sql new file mode 100644 index 0000000..d47ab52 --- /dev/null +++ b/services/migrations/008_add_created_at_users_table.sql @@ -0,0 +1,16 @@ +CREATE OR REPLACE FUNCTION trigger_set_timestamp() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +ALTER TABLE IF EXISTS users +ADD COLUMN created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), +ADD COLUMN updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); + +CREATE TRIGGER set_timestamp +BEFORE UPDATE ON uris +FOR EACH ROW +EXECUTE PROCEDURE trigger_set_timestamp(); From bd07c4b8db0008fd4240d9f65614cb1d5f5be639 Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Wed, 29 May 2019 15:16:53 -0400 Subject: [PATCH 02/14] Adds migration for the site_id column --- postgres/client.js | 18 +++++++++++++----- .../005_add_created_at_uris_table.sql | 4 ++-- .../006_add_created_at_lists_table.sql | 6 +++--- .../007_add_created_at_pages_table.sql | 6 +++--- .../008_add_created_at_users_table.sql | 6 +++--- services/migrations/009_add_site_id_column.sql | 8 ++++++++ services/utils.js | 10 ++++++++++ 7 files changed, 42 insertions(+), 16 deletions(-) create mode 100644 services/migrations/009_add_site_id_column.sql diff --git a/postgres/client.js b/postgres/client.js index 3d55edb..5a72793 100644 --- a/postgres/client.js +++ b/postgres/client.js @@ -2,10 +2,10 @@ const { POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB, CONNECTION_POOL_MIN, CONNECTION_POOL_MAX } = require('../services/constants'), { notFoundError } = require('../services/errors'), - { parseOrNot, wrapInObject, decode } = require('../services/utils'), + { parseOrNot, wrapInObject, decode, isObject } = require('../services/utils'), { findSchemaAndTable, wrapJSONStringInObject } = require('../services/utils'), knexLib = require('knex'), - { isList, isUri } = require('clayutils'), + { isList, isUri, isUser } = require('clayutils'), TransformStream = require('../services/list-transform-stream'), META_PUT_PATCH_FN = patch('meta'); var knex, log = require('../services/log').setup({ file: __filename }); @@ -130,10 +130,19 @@ function columnToValueMap(column, value, obj = {}) { */ function put(key, value) { const { schema, table } = findSchemaAndTable(key), - map = columnToValueMap('id', key); // create the value map + map = columnToValueMap('id', key), // create the value map + parsedValue = parseOrNot(value); + + if (!isUser(key) && isObject(parsedValue) && parsedValue.siteSlug) { + // add site id column to map + columnToValueMap('site_id', parsedValue.siteSlug, map); + + // delete this property to avoid duplication in data column + delete parsedValue.siteSlug; + } // add data to the map - columnToValueMap('data', wrapInObject(key, parseOrNot(value)), map); + columnToValueMap('data', wrapInObject(key, parsedValue), map); let url; @@ -144,7 +153,6 @@ function put(key, value) { columnToValueMap('url', url, map); } - return onConflictPut(map, schema, table) .then(() => map.data); } diff --git a/services/migrations/005_add_created_at_uris_table.sql b/services/migrations/005_add_created_at_uris_table.sql index 3cfbf28..b97290a 100644 --- a/services/migrations/005_add_created_at_uris_table.sql +++ b/services/migrations/005_add_created_at_uris_table.sql @@ -7,8 +7,8 @@ END; $$ LANGUAGE plpgsql; ALTER TABLE IF EXISTS uris -ADD COLUMN created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), -ADD COLUMN updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); CREATE TRIGGER set_timestamp BEFORE UPDATE ON uris diff --git a/services/migrations/006_add_created_at_lists_table.sql b/services/migrations/006_add_created_at_lists_table.sql index e450f6d..9902917 100644 --- a/services/migrations/006_add_created_at_lists_table.sql +++ b/services/migrations/006_add_created_at_lists_table.sql @@ -7,10 +7,10 @@ END; $$ LANGUAGE plpgsql; ALTER TABLE IF EXISTS lists -ADD COLUMN created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), -ADD COLUMN updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); CREATE TRIGGER set_timestamp -BEFORE UPDATE ON uris +BEFORE UPDATE ON lists FOR EACH ROW EXECUTE PROCEDURE trigger_set_timestamp(); diff --git a/services/migrations/007_add_created_at_pages_table.sql b/services/migrations/007_add_created_at_pages_table.sql index f2e496f..b33db2c 100644 --- a/services/migrations/007_add_created_at_pages_table.sql +++ b/services/migrations/007_add_created_at_pages_table.sql @@ -7,10 +7,10 @@ END; $$ LANGUAGE plpgsql; ALTER TABLE IF EXISTS pages -ADD COLUMN created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), -ADD COLUMN updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); CREATE TRIGGER set_timestamp -BEFORE UPDATE ON uris +BEFORE UPDATE ON pages FOR EACH ROW EXECUTE PROCEDURE trigger_set_timestamp(); diff --git a/services/migrations/008_add_created_at_users_table.sql b/services/migrations/008_add_created_at_users_table.sql index d47ab52..095903f 100644 --- a/services/migrations/008_add_created_at_users_table.sql +++ b/services/migrations/008_add_created_at_users_table.sql @@ -7,10 +7,10 @@ END; $$ LANGUAGE plpgsql; ALTER TABLE IF EXISTS users -ADD COLUMN created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), -ADD COLUMN updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); CREATE TRIGGER set_timestamp -BEFORE UPDATE ON uris +BEFORE UPDATE ON users FOR EACH ROW EXECUTE PROCEDURE trigger_set_timestamp(); diff --git a/services/migrations/009_add_site_id_column.sql b/services/migrations/009_add_site_id_column.sql new file mode 100644 index 0000000..f9ee5d8 --- /dev/null +++ b/services/migrations/009_add_site_id_column.sql @@ -0,0 +1,8 @@ +ALTER TABLE IF EXISTS pages +ADD COLUMN IF NOT EXISTS site_id; + +ALTER TABLE IF EXISTS uris +ADD COLUMN IF NOT EXISTS site_id; + +ALTER TABLE IF EXISTS lists +ADD COLUMN IF NOT EXISTS site_id; diff --git a/services/utils.js b/services/utils.js index a08b0c0..3e87887 100644 --- a/services/utils.js +++ b/services/utils.js @@ -78,8 +78,18 @@ function decode(string) { return Buffer.from(string, 'base64').toString('utf8'); } +/** + * Check whether the passed value is an Object or not. + * @param {any} value + * @returns {boolean} + */ +function isObject(value) { + return typeof value === 'object' && !Array.isArray(value) && value !== null && !!value; +} + module.exports.findSchemaAndTable = findSchemaAndTable; module.exports.parseOrNot = parseOrNot; module.exports.wrapInObject = wrapInObject; module.exports.wrapJSONStringInObject = wrapJSONStringInObject; module.exports.decode = decode; +module.exports.isObject = isObject; From a696762eb7090df4543c1b89f82767ec52676270 Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Wed, 29 May 2019 17:45:16 -0400 Subject: [PATCH 03/14] Fixes error in site id column migration --- postgres/index.js | 2 +- services/migrations/009_add_site_id_column.sql | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/postgres/index.js b/postgres/index.js index 6b7322b..c1e1e20 100644 --- a/postgres/index.js +++ b/postgres/index.js @@ -76,7 +76,7 @@ function setup(testPostgresHost) { }) .then(() => createTables()) .then(() => ({ server: `${postgresHost}:${POSTGRES_PORT}` })) - .catch(logGenericError); + .catch(logGenericError(__filename)); } module.exports.setup = setup; diff --git a/services/migrations/009_add_site_id_column.sql b/services/migrations/009_add_site_id_column.sql index f9ee5d8..509a12b 100644 --- a/services/migrations/009_add_site_id_column.sql +++ b/services/migrations/009_add_site_id_column.sql @@ -1,8 +1,8 @@ ALTER TABLE IF EXISTS pages -ADD COLUMN IF NOT EXISTS site_id; +ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); ALTER TABLE IF EXISTS uris -ADD COLUMN IF NOT EXISTS site_id; +ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); ALTER TABLE IF EXISTS lists -ADD COLUMN IF NOT EXISTS site_id; +ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); From 498d70287d92b85ad837d131b852c0e61e23c124 Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Thu, 30 May 2019 17:33:04 -0400 Subject: [PATCH 04/14] Modifies migration script and put logic --- postgres/client.js | 11 ++++------- services/migrations/009_add_site_id_column.sql | 8 -------- services/migrations/009_add_site_id_column_pages.sql | 5 +++++ services/migrations/010_add_site_id_column_uris.sql | 5 +++++ services/migrations/011_add_site_id_column_lists.sql | 5 +++++ 5 files changed, 19 insertions(+), 15 deletions(-) delete mode 100644 services/migrations/009_add_site_id_column.sql create mode 100644 services/migrations/009_add_site_id_column_pages.sql create mode 100644 services/migrations/010_add_site_id_column_uris.sql create mode 100644 services/migrations/011_add_site_id_column_lists.sql diff --git a/postgres/client.js b/postgres/client.js index 5a72793..16aec2c 100644 --- a/postgres/client.js +++ b/postgres/client.js @@ -2,10 +2,10 @@ const { POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB, CONNECTION_POOL_MIN, CONNECTION_POOL_MAX } = require('../services/constants'), { notFoundError } = require('../services/errors'), - { parseOrNot, wrapInObject, decode, isObject } = require('../services/utils'), + { parseOrNot, wrapInObject, decode } = require('../services/utils'), { findSchemaAndTable, wrapJSONStringInObject } = require('../services/utils'), knexLib = require('knex'), - { isList, isUri, isUser } = require('clayutils'), + { isList, isUri, isUser, getPrefix } = require('clayutils'), TransformStream = require('../services/list-transform-stream'), META_PUT_PATCH_FN = patch('meta'); var knex, log = require('../services/log').setup({ file: __filename }); @@ -133,12 +133,9 @@ function put(key, value) { map = columnToValueMap('id', key), // create the value map parsedValue = parseOrNot(value); - if (!isUser(key) && isObject(parsedValue) && parsedValue.siteSlug) { + if (!isUser(key)) { // add site id column to map - columnToValueMap('site_id', parsedValue.siteSlug, map); - - // delete this property to avoid duplication in data column - delete parsedValue.siteSlug; + columnToValueMap('site_id', getPrefix(key), map); } // add data to the map diff --git a/services/migrations/009_add_site_id_column.sql b/services/migrations/009_add_site_id_column.sql deleted file mode 100644 index 509a12b..0000000 --- a/services/migrations/009_add_site_id_column.sql +++ /dev/null @@ -1,8 +0,0 @@ -ALTER TABLE IF EXISTS pages -ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); - -ALTER TABLE IF EXISTS uris -ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); - -ALTER TABLE IF EXISTS lists -ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); diff --git a/services/migrations/009_add_site_id_column_pages.sql b/services/migrations/009_add_site_id_column_pages.sql new file mode 100644 index 0000000..e0a68fe --- /dev/null +++ b/services/migrations/009_add_site_id_column_pages.sql @@ -0,0 +1,5 @@ +ALTER TABLE IF EXISTS pages +ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); + +UPDATE pages +SET site_id = (SELECT LEFT(id, STRPOS(id, '/_pages') - 1) FROM pages LIMIT 1); diff --git a/services/migrations/010_add_site_id_column_uris.sql b/services/migrations/010_add_site_id_column_uris.sql new file mode 100644 index 0000000..f7b61f0 --- /dev/null +++ b/services/migrations/010_add_site_id_column_uris.sql @@ -0,0 +1,5 @@ +ALTER TABLE IF EXISTS uris +ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); + +UPDATE uris +SET site_id = (SELECT LEFT(id, STRPOS(id, '/_uris') - 1) FROM uris LIMIT 1); diff --git a/services/migrations/011_add_site_id_column_lists.sql b/services/migrations/011_add_site_id_column_lists.sql new file mode 100644 index 0000000..2daec21 --- /dev/null +++ b/services/migrations/011_add_site_id_column_lists.sql @@ -0,0 +1,5 @@ +ALTER TABLE IF EXISTS lists +ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); + +UPDATE lists +SET site_id = (SELECT LEFT(id, STRPOS(id, '/_lists') - 1) FROM lists LIMIT 1); From 47e010ae63ca07a299290b15d35193735670edfa Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Tue, 4 Jun 2019 14:30:06 -0400 Subject: [PATCH 05/14] Fixes issue that prevent pages from publishing and adding the site_id value --- postgres/client.js | 23 +++++++++++++------ .../009_add_site_id_column_pages.sql | 3 --- .../010_add_site_id_column_uris.sql | 3 --- .../011_add_site_id_column_lists.sql | 3 --- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/postgres/client.js b/postgres/client.js index 16aec2c..a574e49 100644 --- a/postgres/client.js +++ b/postgres/client.js @@ -5,7 +5,7 @@ const { POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, POSTGRES_PORT, POSTGRES { parseOrNot, wrapInObject, decode } = require('../services/utils'), { findSchemaAndTable, wrapJSONStringInObject } = require('../services/utils'), knexLib = require('knex'), - { isList, isUri, isUser, getPrefix } = require('clayutils'), + { isList, isUri, isPage } = require('clayutils'), TransformStream = require('../services/list-transform-stream'), META_PUT_PATCH_FN = patch('meta'); var knex, log = require('../services/log').setup({ file: __filename }); @@ -131,15 +131,16 @@ function columnToValueMap(column, value, obj = {}) { function put(key, value) { const { schema, table } = findSchemaAndTable(key), map = columnToValueMap('id', key), // create the value map + isPublicEntity = isPage(key) || isList(key) || isUri(key), parsedValue = parseOrNot(value); - if (!isUser(key)) { + if (isPublicEntity && parsedValue.siteSlug) { // add site id column to map - columnToValueMap('site_id', getPrefix(key), map); + columnToValueMap('site_id', parsedValue.siteSlug, map); } // add data to the map - columnToValueMap('data', wrapInObject(key, parsedValue), map); + columnToValueMap('data', wrapInObject(key, parsedValue.data), map); let url; @@ -212,15 +213,23 @@ function del(key) { * @param {[type]} ops [description] * @return {[type]} [description] */ +/* eslint complexity: 0 */ function batch(ops) { var commands = [], url; - for (let i = 0; i < ops.length; i++) { + for (let i = 0, opsLength = ops.length; i < opsLength; i++) { let { key, value } = ops[i], { table, schema } = findSchemaAndTable(key), - map = columnToValueMap('id', key); + map = columnToValueMap('id', key), + isPublicEntity = isPage(key) || isList(key) || isUri(key), + parsedValue = isPublicEntity ? parseOrNot(value) : { data: value }; - columnToValueMap('data', wrapJSONStringInObject(key, value), map); + if (isPublicEntity && parsedValue.siteSlug) { + // add site id column to map + columnToValueMap('site_id', parsedValue.siteSlug, map); + } + + columnToValueMap('data', wrapJSONStringInObject(key, parsedValue.data), map); // add url column to map if putting a uri if (isUri(key)) { diff --git a/services/migrations/009_add_site_id_column_pages.sql b/services/migrations/009_add_site_id_column_pages.sql index e0a68fe..07debf1 100644 --- a/services/migrations/009_add_site_id_column_pages.sql +++ b/services/migrations/009_add_site_id_column_pages.sql @@ -1,5 +1,2 @@ ALTER TABLE IF EXISTS pages ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); - -UPDATE pages -SET site_id = (SELECT LEFT(id, STRPOS(id, '/_pages') - 1) FROM pages LIMIT 1); diff --git a/services/migrations/010_add_site_id_column_uris.sql b/services/migrations/010_add_site_id_column_uris.sql index f7b61f0..e04b8b7 100644 --- a/services/migrations/010_add_site_id_column_uris.sql +++ b/services/migrations/010_add_site_id_column_uris.sql @@ -1,5 +1,2 @@ ALTER TABLE IF EXISTS uris ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); - -UPDATE uris -SET site_id = (SELECT LEFT(id, STRPOS(id, '/_uris') - 1) FROM uris LIMIT 1); diff --git a/services/migrations/011_add_site_id_column_lists.sql b/services/migrations/011_add_site_id_column_lists.sql index 2daec21..d7d3e61 100644 --- a/services/migrations/011_add_site_id_column_lists.sql +++ b/services/migrations/011_add_site_id_column_lists.sql @@ -1,5 +1,2 @@ ALTER TABLE IF EXISTS lists ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); - -UPDATE lists -SET site_id = (SELECT LEFT(id, STRPOS(id, '/_lists') - 1) FROM lists LIMIT 1); From d1113fc026bd82a8e9eb418441fdc35dc42b84a4 Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Tue, 4 Jun 2019 15:00:42 -0400 Subject: [PATCH 06/14] Updates test cases that were affected by the changes: --- postgres/client.test.js | 27 ++++++++++++++++++++++++--- services/utils.js | 10 ---------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/postgres/client.test.js b/postgres/client.test.js index 0ce693b..7c7b5db 100644 --- a/postgres/client.test.js +++ b/postgres/client.test.js @@ -566,6 +566,27 @@ describe('postgres/client', () => { expect(data).toEqual(data); }); }); + + test('inserts a row with a column for site id', () => { + const key = 'nymag.com/_pages/sample-article', + tableName = 'pages', + putData = { data, siteSlug: 'nymag' }; + + client.setClient(knex); + + return client.put(key, putData).then((data) => { + expect(table.mock.calls.length).toBe(1); + expect(table.mock.calls[0][0]).toBe(tableName); + expect(insert.mock.calls.length).toBe(1); + expect(insert.mock.calls[0][0]).toEqual({ id: key, data, site_id: 'nymag' }); + expect(queryBuilder.mock.calls.length).toBe(1); + expect(update.mock.calls.length).toBe(1); + expect(raw.mock.calls.length).toBe(1); + expect(raw.mock.calls[0][0]).toBe('? ON CONFLICT (id) DO ? returning *'); + expect(raw.mock.calls[0][1]).toEqual(['insert sql', 'update sql']); + expect(putData.data).toEqual(data); + }); + }); }); describe('putMeta', () => { @@ -674,11 +695,11 @@ describe('postgres/client', () => { ops = [ { key: 'nymag.com/_uris/someinstance', - value: 'nymag.com/_pages/someinstance' + value: { data: 'nymag.com/_pages/someinstance', siteSlug: 'nymag' } }, { key: 'nymag.com/_uris/someotherinstance', - value: 'nymag.com/_pages/someotherinstance' + value: { data: 'nymag.com/_pages/someotherinstance', siteSlug: 'nymag' } } ]; @@ -694,7 +715,7 @@ describe('postgres/client', () => { for (let index = 0; index < results.length; index++) { expect(table.mock.calls[index][0]).toBe('uris'); - expect(insert.mock.calls[index][0]).toEqual({ id: ops[index].key, data: results[index], url: decode(ops[index].key.split('/_uris/').pop()) }); + expect(insert.mock.calls[index][0]).toEqual({ id: ops[index].key, data: results[index], url: decode(ops[index].key.split('/_uris/').pop()), site_id: 'nymag' }); expect(raw.mock.calls[index][0]).toBe('? ON CONFLICT (id) DO ? returning *'); expect(raw.mock.calls[index][1]).toEqual(['insert sql', 'update sql']); } diff --git a/services/utils.js b/services/utils.js index 3e87887..a08b0c0 100644 --- a/services/utils.js +++ b/services/utils.js @@ -78,18 +78,8 @@ function decode(string) { return Buffer.from(string, 'base64').toString('utf8'); } -/** - * Check whether the passed value is an Object or not. - * @param {any} value - * @returns {boolean} - */ -function isObject(value) { - return typeof value === 'object' && !Array.isArray(value) && value !== null && !!value; -} - module.exports.findSchemaAndTable = findSchemaAndTable; module.exports.parseOrNot = parseOrNot; module.exports.wrapInObject = wrapInObject; module.exports.wrapJSONStringInObject = wrapJSONStringInObject; module.exports.decode = decode; -module.exports.isObject = isObject; From ba398ff175436e4ba746f6a0a0ee4366358bbf13 Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Tue, 11 Jun 2019 16:08:42 -0400 Subject: [PATCH 07/14] Fixes issue that prevented pages from unpublishing --- redis/index.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/redis/index.js b/redis/index.js index c5f84a8..4611cc0 100644 --- a/redis/index.js +++ b/redis/index.js @@ -4,7 +4,8 @@ const bluebird = require('bluebird'), Redis = require('ioredis'), { REDIS_URL, REDIS_HASH } = require('../services/constants'), { isPublished, isUri, isUser } = require('clayutils'), - { notFoundError, logGenericError } = require('../services/errors'); + { notFoundError, logGenericError } = require('../services/errors'), + { parseOrNot } = require('../services/utils'); var log = require('../services/log').setup({ file: __filename }); /** @@ -50,7 +51,11 @@ function shouldProcess(key) { function put(key, value) { if (!shouldProcess(key)) return bluebird.resolve(); - return module.exports.client.hsetAsync(REDIS_HASH, key, value); + const data = isUri(key) + ? parseOrNot(value).data || value + : value; + + return module.exports.client.hsetAsync(REDIS_HASH, key, data); } /** @@ -69,7 +74,7 @@ function get(key) { } /** - * [batch description] + * Makes a batch operation to redis * @param {[type]} ops * @return {[type]} */ @@ -84,8 +89,12 @@ function batch(ops) { let { key, value } = ops[i]; if (shouldProcess(key)) { + const data = isUri(key) + ? parseOrNot(value).data || value + : value; + batch.push(key); - batch.push(value); + batch.push(data); } } From 58d9c5657c947a583e1f3144d8da0801bc2b3488 Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Tue, 11 Jun 2019 17:23:44 -0400 Subject: [PATCH 08/14] Fixes user data not being stored --- postgres/client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/client.js b/postgres/client.js index a574e49..033ffea 100644 --- a/postgres/client.js +++ b/postgres/client.js @@ -140,7 +140,7 @@ function put(key, value) { } // add data to the map - columnToValueMap('data', wrapInObject(key, parsedValue.data), map); + columnToValueMap('data', wrapInObject(key, parsedValue.data || parsedValue), map); let url; From 51fd22d44f8aefa77b1470270fb7b291c171a51d Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Wed, 26 Jun 2019 14:05:04 -0400 Subject: [PATCH 09/14] Applies PR code review feedback --- services/migrations/005_add_created_at_uris_table.sql | 4 ++-- services/migrations/006_add_created_at_lists_table.sql | 4 ++-- services/migrations/007_add_created_at_pages_table.sql | 4 ++-- services/migrations/008_add_created_at_users_table.sql | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/services/migrations/005_add_created_at_uris_table.sql b/services/migrations/005_add_created_at_uris_table.sql index b97290a..f129be0 100644 --- a/services/migrations/005_add_created_at_uris_table.sql +++ b/services/migrations/005_add_created_at_uris_table.sql @@ -7,8 +7,8 @@ END; $$ LANGUAGE plpgsql; ALTER TABLE IF EXISTS uris -ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), -ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(), +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); CREATE TRIGGER set_timestamp BEFORE UPDATE ON uris diff --git a/services/migrations/006_add_created_at_lists_table.sql b/services/migrations/006_add_created_at_lists_table.sql index 9902917..e3505b7 100644 --- a/services/migrations/006_add_created_at_lists_table.sql +++ b/services/migrations/006_add_created_at_lists_table.sql @@ -7,8 +7,8 @@ END; $$ LANGUAGE plpgsql; ALTER TABLE IF EXISTS lists -ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), -ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(), +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); CREATE TRIGGER set_timestamp BEFORE UPDATE ON lists diff --git a/services/migrations/007_add_created_at_pages_table.sql b/services/migrations/007_add_created_at_pages_table.sql index b33db2c..8697dc1 100644 --- a/services/migrations/007_add_created_at_pages_table.sql +++ b/services/migrations/007_add_created_at_pages_table.sql @@ -7,8 +7,8 @@ END; $$ LANGUAGE plpgsql; ALTER TABLE IF EXISTS pages -ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), -ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(), +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); CREATE TRIGGER set_timestamp BEFORE UPDATE ON pages diff --git a/services/migrations/008_add_created_at_users_table.sql b/services/migrations/008_add_created_at_users_table.sql index 095903f..f48900d 100644 --- a/services/migrations/008_add_created_at_users_table.sql +++ b/services/migrations/008_add_created_at_users_table.sql @@ -7,8 +7,8 @@ END; $$ LANGUAGE plpgsql; ALTER TABLE IF EXISTS users -ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), -ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(), +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); CREATE TRIGGER set_timestamp BEFORE UPDATE ON users From 8f07b360b020660649d3e36006c3767d9860794f Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Thu, 27 Jun 2019 10:54:00 -0400 Subject: [PATCH 10/14] Modifies migration run time --- postgres/index.js | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/postgres/index.js b/postgres/index.js index c1e1e20..0476eff 100644 --- a/postgres/index.js +++ b/postgres/index.js @@ -59,22 +59,20 @@ function setup(testPostgresHost) { } return client.connect() - .then(() => { - return migrate( - { - database: POSTGRES_DB, - user: POSTGRES_USER, - password: POSTGRES_PASSWORD, - host: postgresHost, - port: POSTGRES_PORT - }, - path.join(__dirname, '../services/migrations') - ); - }) - .then(() => { - log('info', 'Migrations Complete'); - }) - .then(() => createTables()) + .then(() => client.createSchema('components')) + .then(() => client.createSchema('layouts')) + .then(createTables) + .then(() => migrate( + { + database: POSTGRES_DB, + user: POSTGRES_USER, + password: POSTGRES_PASSWORD, + host: postgresHost, + port: POSTGRES_PORT + }, + path.join(__dirname, '../services/migrations') + )) + .then(() => log('info', 'Migrations Complete')) .then(() => ({ server: `${postgresHost}:${POSTGRES_PORT}` })) .catch(logGenericError(__filename)); } From f8c2556b6aa5daafe0c868fbb5de26ea74accb9b Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Thu, 27 Jun 2019 11:32:28 -0400 Subject: [PATCH 11/14] Removes sql queries to create schemas and tables --- services/migrations/001_create_components_schema.sql | 1 - services/migrations/001_create_timestamp_trigger.sql | 7 +++++++ ...t_uris_table.sql => 002_add_created_at_uris_table.sql} | 8 -------- services/migrations/002_create_layouts_schema.sql | 1 - ...lists_table.sql => 003_add_created_at_lists_table.sql} | 0 services/migrations/003_create_pages_table.sql | 1 - ...pages_table.sql => 004_add_created_at_pages_table.sql} | 8 -------- services/migrations/004_create_uris_table.sql | 1 - ...users_table.sql => 005_add_created_at_users_table.sql} | 8 -------- ..._column_pages.sql => 006_add_site_id_column_pages.sql} | 0 ..._column_lists.sql => 007_add_site_id_column_lists.sql} | 0 ...id_column_uris.sql => 008_add_site_id_column_uris.sql} | 0 12 files changed, 7 insertions(+), 28 deletions(-) delete mode 100644 services/migrations/001_create_components_schema.sql create mode 100644 services/migrations/001_create_timestamp_trigger.sql rename services/migrations/{005_add_created_at_uris_table.sql => 002_add_created_at_uris_table.sql} (64%) delete mode 100644 services/migrations/002_create_layouts_schema.sql rename services/migrations/{006_add_created_at_lists_table.sql => 003_add_created_at_lists_table.sql} (100%) delete mode 100644 services/migrations/003_create_pages_table.sql rename services/migrations/{007_add_created_at_pages_table.sql => 004_add_created_at_pages_table.sql} (64%) delete mode 100644 services/migrations/004_create_uris_table.sql rename services/migrations/{008_add_created_at_users_table.sql => 005_add_created_at_users_table.sql} (64%) rename services/migrations/{009_add_site_id_column_pages.sql => 006_add_site_id_column_pages.sql} (100%) rename services/migrations/{011_add_site_id_column_lists.sql => 007_add_site_id_column_lists.sql} (100%) rename services/migrations/{010_add_site_id_column_uris.sql => 008_add_site_id_column_uris.sql} (100%) diff --git a/services/migrations/001_create_components_schema.sql b/services/migrations/001_create_components_schema.sql deleted file mode 100644 index 042d10e..0000000 --- a/services/migrations/001_create_components_schema.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE SCHEMA IF NOT EXISTS components; diff --git a/services/migrations/001_create_timestamp_trigger.sql b/services/migrations/001_create_timestamp_trigger.sql new file mode 100644 index 0000000..cc917c3 --- /dev/null +++ b/services/migrations/001_create_timestamp_trigger.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE FUNCTION trigger_set_timestamp() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/services/migrations/005_add_created_at_uris_table.sql b/services/migrations/002_add_created_at_uris_table.sql similarity index 64% rename from services/migrations/005_add_created_at_uris_table.sql rename to services/migrations/002_add_created_at_uris_table.sql index f129be0..3f57b4b 100644 --- a/services/migrations/005_add_created_at_uris_table.sql +++ b/services/migrations/002_add_created_at_uris_table.sql @@ -1,11 +1,3 @@ -CREATE OR REPLACE FUNCTION trigger_set_timestamp() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - ALTER TABLE IF EXISTS uris ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(), ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); diff --git a/services/migrations/002_create_layouts_schema.sql b/services/migrations/002_create_layouts_schema.sql deleted file mode 100644 index 8a3e1dc..0000000 --- a/services/migrations/002_create_layouts_schema.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE SCHEMA IF NOT EXISTS layouts; diff --git a/services/migrations/006_add_created_at_lists_table.sql b/services/migrations/003_add_created_at_lists_table.sql similarity index 100% rename from services/migrations/006_add_created_at_lists_table.sql rename to services/migrations/003_add_created_at_lists_table.sql diff --git a/services/migrations/003_create_pages_table.sql b/services/migrations/003_create_pages_table.sql deleted file mode 100644 index 8bb3792..0000000 --- a/services/migrations/003_create_pages_table.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE TABLE IF NOT EXISTS "pages" ( id TEXT PRIMARY KEY NOT NULL, data JSONB, meta JSONB ); diff --git a/services/migrations/007_add_created_at_pages_table.sql b/services/migrations/004_add_created_at_pages_table.sql similarity index 64% rename from services/migrations/007_add_created_at_pages_table.sql rename to services/migrations/004_add_created_at_pages_table.sql index 8697dc1..cfbe515 100644 --- a/services/migrations/007_add_created_at_pages_table.sql +++ b/services/migrations/004_add_created_at_pages_table.sql @@ -1,11 +1,3 @@ -CREATE OR REPLACE FUNCTION trigger_set_timestamp() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - ALTER TABLE IF EXISTS pages ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(), ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); diff --git a/services/migrations/004_create_uris_table.sql b/services/migrations/004_create_uris_table.sql deleted file mode 100644 index 2d74761..0000000 --- a/services/migrations/004_create_uris_table.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE TABLE IF NOT EXISTS "uris" ( id TEXT PRIMARY KEY NOT NULL, data TEXT NOT NULL, url TEXT ); diff --git a/services/migrations/008_add_created_at_users_table.sql b/services/migrations/005_add_created_at_users_table.sql similarity index 64% rename from services/migrations/008_add_created_at_users_table.sql rename to services/migrations/005_add_created_at_users_table.sql index f48900d..1bc971d 100644 --- a/services/migrations/008_add_created_at_users_table.sql +++ b/services/migrations/005_add_created_at_users_table.sql @@ -1,11 +1,3 @@ -CREATE OR REPLACE FUNCTION trigger_set_timestamp() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - ALTER TABLE IF EXISTS users ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(), ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); diff --git a/services/migrations/009_add_site_id_column_pages.sql b/services/migrations/006_add_site_id_column_pages.sql similarity index 100% rename from services/migrations/009_add_site_id_column_pages.sql rename to services/migrations/006_add_site_id_column_pages.sql diff --git a/services/migrations/011_add_site_id_column_lists.sql b/services/migrations/007_add_site_id_column_lists.sql similarity index 100% rename from services/migrations/011_add_site_id_column_lists.sql rename to services/migrations/007_add_site_id_column_lists.sql diff --git a/services/migrations/010_add_site_id_column_uris.sql b/services/migrations/008_add_site_id_column_uris.sql similarity index 100% rename from services/migrations/010_add_site_id_column_uris.sql rename to services/migrations/008_add_site_id_column_uris.sql From bddc24a5628270e2af0c5cf329b98d4198e0b8c6 Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Thu, 27 Jun 2019 11:43:32 -0400 Subject: [PATCH 12/14] Removes uris condition --- postgres/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/index.js b/postgres/index.js index 0476eff..95afd4b 100644 --- a/postgres/index.js +++ b/postgres/index.js @@ -25,7 +25,7 @@ function createRemainingTables() { for (let i = 0; i < DATA_STRUCTURES.length; i++) { let STRUCTURE = DATA_STRUCTURES[i]; - if (STRUCTURE !== 'components' && STRUCTURE !== 'pages' && STRUCTURE !== 'layouts' && STRUCTURE !== 'uris') { + if (STRUCTURE !== 'components' && STRUCTURE !== 'pages' && STRUCTURE !== 'layouts') { promises.push(client.createTable(STRUCTURE)); } } From 345017cc934b9f0690b3fa65ee68dc6af74b7590 Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Thu, 27 Jun 2019 15:27:37 -0400 Subject: [PATCH 13/14] Adds uris table creation statement --- postgres/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/postgres/index.js b/postgres/index.js index 95afd4b..9dce850 100644 --- a/postgres/index.js +++ b/postgres/index.js @@ -25,7 +25,7 @@ function createRemainingTables() { for (let i = 0; i < DATA_STRUCTURES.length; i++) { let STRUCTURE = DATA_STRUCTURES[i]; - if (STRUCTURE !== 'components' && STRUCTURE !== 'pages' && STRUCTURE !== 'layouts') { + if (STRUCTURE !== 'components' && STRUCTURE !== 'pages' && STRUCTURE !== 'layouts' && STRUCTURE !== 'uris') { promises.push(client.createTable(STRUCTURE)); } } @@ -42,6 +42,7 @@ function createTables() { return bluebird.all(getComponents().map(component => client.createTable(`components.${component}`))) .then(() => bluebird.all(getLayouts().map(layout => client.createTableWithMeta(`layouts.${layout}`)))) .then(() => client.createTableWithMeta('pages')) + .then(() => client.raw('CREATE TABLE IF NOT EXISTS ?? ( id TEXT PRIMARY KEY NOT NULL, data TEXT NOT NULL, url TEXT );', ['uris'])) .then(() => createRemainingTables()); } From 5d2076e4a69da2c3720614c10914580cec60e51e Mon Sep 17 00:00:00 2001 From: Manuel Emilio Urena Date: Thu, 4 Jul 2019 17:27:41 -0400 Subject: [PATCH 14/14] Updates migration queries --- .../002_add_created_at_uris_table.sql | 8 -------- .../002_add_created_updated_columns.sql | 15 +++++++++++++++ .../003_add_created_at_lists_table.sql | 16 ---------------- .../migrations/003_add_update_trigger.sql | 19 +++++++++++++++++++ .../004_add_created_at_pages_table.sql | 8 -------- .../004_add_created_at_values_pages.sql | 7 +++++++ .../005_add_created_at_users_table.sql | 8 -------- .../005_add_updated_at_values_pages.sql | 7 +++++++ ...006_add_default_created_updated_values.sql | 15 +++++++++++++++ .../006_add_site_id_column_pages.sql | 2 -- .../migrations/007_add_site_id_column.sql | 8 ++++++++ .../007_add_site_id_column_lists.sql | 2 -- .../008_add_site_id_column_uris.sql | 2 -- .../008_add_site_id_values_pages.sql | 7 +++++++ .../009_add_site_id_values_uris.sql | 7 +++++++ 15 files changed, 85 insertions(+), 46 deletions(-) delete mode 100644 services/migrations/002_add_created_at_uris_table.sql create mode 100644 services/migrations/002_add_created_updated_columns.sql delete mode 100644 services/migrations/003_add_created_at_lists_table.sql create mode 100644 services/migrations/003_add_update_trigger.sql delete mode 100644 services/migrations/004_add_created_at_pages_table.sql create mode 100644 services/migrations/004_add_created_at_values_pages.sql delete mode 100644 services/migrations/005_add_created_at_users_table.sql create mode 100644 services/migrations/005_add_updated_at_values_pages.sql create mode 100644 services/migrations/006_add_default_created_updated_values.sql delete mode 100644 services/migrations/006_add_site_id_column_pages.sql create mode 100644 services/migrations/007_add_site_id_column.sql delete mode 100644 services/migrations/007_add_site_id_column_lists.sql delete mode 100644 services/migrations/008_add_site_id_column_uris.sql create mode 100644 services/migrations/008_add_site_id_values_pages.sql create mode 100644 services/migrations/009_add_site_id_values_uris.sql diff --git a/services/migrations/002_add_created_at_uris_table.sql b/services/migrations/002_add_created_at_uris_table.sql deleted file mode 100644 index 3f57b4b..0000000 --- a/services/migrations/002_add_created_at_uris_table.sql +++ /dev/null @@ -1,8 +0,0 @@ -ALTER TABLE IF EXISTS uris -ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(), -ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); - -CREATE TRIGGER set_timestamp -BEFORE UPDATE ON uris -FOR EACH ROW -EXECUTE PROCEDURE trigger_set_timestamp(); diff --git a/services/migrations/002_add_created_updated_columns.sql b/services/migrations/002_add_created_updated_columns.sql new file mode 100644 index 0000000..a62a5da --- /dev/null +++ b/services/migrations/002_add_created_updated_columns.sql @@ -0,0 +1,15 @@ +ALTER TABLE IF EXISTS uris +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ, +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ; + +ALTER TABLE IF EXISTS lists +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ, +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ; + +ALTER TABLE IF EXISTS pages +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ, +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ; + +ALTER TABLE IF EXISTS users +ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ, +ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ; diff --git a/services/migrations/003_add_created_at_lists_table.sql b/services/migrations/003_add_created_at_lists_table.sql deleted file mode 100644 index e3505b7..0000000 --- a/services/migrations/003_add_created_at_lists_table.sql +++ /dev/null @@ -1,16 +0,0 @@ -CREATE OR REPLACE FUNCTION trigger_set_timestamp() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = NOW(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -ALTER TABLE IF EXISTS lists -ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(), -ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); - -CREATE TRIGGER set_timestamp -BEFORE UPDATE ON lists -FOR EACH ROW -EXECUTE PROCEDURE trigger_set_timestamp(); diff --git a/services/migrations/003_add_update_trigger.sql b/services/migrations/003_add_update_trigger.sql new file mode 100644 index 0000000..8df32ff --- /dev/null +++ b/services/migrations/003_add_update_trigger.sql @@ -0,0 +1,19 @@ +CREATE TRIGGER set_timestamp +BEFORE UPDATE ON lists +FOR EACH ROW +EXECUTE PROCEDURE trigger_set_timestamp(); + +CREATE TRIGGER set_timestamp +BEFORE UPDATE ON uris +FOR EACH ROW +EXECUTE PROCEDURE trigger_set_timestamp(); + +CREATE TRIGGER set_timestamp +BEFORE UPDATE ON pages +FOR EACH ROW +EXECUTE PROCEDURE trigger_set_timestamp(); + +CREATE TRIGGER set_timestamp +BEFORE UPDATE ON users +FOR EACH ROW +EXECUTE PROCEDURE trigger_set_timestamp(); diff --git a/services/migrations/004_add_created_at_pages_table.sql b/services/migrations/004_add_created_at_pages_table.sql deleted file mode 100644 index cfbe515..0000000 --- a/services/migrations/004_add_created_at_pages_table.sql +++ /dev/null @@ -1,8 +0,0 @@ -ALTER TABLE IF EXISTS pages -ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(), -ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); - -CREATE TRIGGER set_timestamp -BEFORE UPDATE ON pages -FOR EACH ROW -EXECUTE PROCEDURE trigger_set_timestamp(); diff --git a/services/migrations/004_add_created_at_values_pages.sql b/services/migrations/004_add_created_at_values_pages.sql new file mode 100644 index 0000000..a5f80dc --- /dev/null +++ b/services/migrations/004_add_created_at_values_pages.sql @@ -0,0 +1,7 @@ +UPDATE pages +SET created_at = subquery.created_at +FROM (SELECT id, meta ->> 'createdAt' as created_at + FROM pages + WHERE meta IS NOT NULL + AND meta ->> 'createdAt' IS NOT NULL) AS subquery +WHERE pages.id = subquery.id; diff --git a/services/migrations/005_add_created_at_users_table.sql b/services/migrations/005_add_created_at_users_table.sql deleted file mode 100644 index 1bc971d..0000000 --- a/services/migrations/005_add_created_at_users_table.sql +++ /dev/null @@ -1,8 +0,0 @@ -ALTER TABLE IF EXISTS users -ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT NOW(), -ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW(); - -CREATE TRIGGER set_timestamp -BEFORE UPDATE ON users -FOR EACH ROW -EXECUTE PROCEDURE trigger_set_timestamp(); diff --git a/services/migrations/005_add_updated_at_values_pages.sql b/services/migrations/005_add_updated_at_values_pages.sql new file mode 100644 index 0000000..f76fd0f --- /dev/null +++ b/services/migrations/005_add_updated_at_values_pages.sql @@ -0,0 +1,7 @@ +UPDATE pages +SET updated_at = subquery.updated_at +FROM (SELECT id, meta ->> 'updateTime' as updated_at + FROM pages + WHERE meta IS NOT NULL + AND meta ->> 'updateTime' IS NOT NULL) AS subquery +WHERE pages.id = subquery.id; diff --git a/services/migrations/006_add_default_created_updated_values.sql b/services/migrations/006_add_default_created_updated_values.sql new file mode 100644 index 0000000..3d7f53b --- /dev/null +++ b/services/migrations/006_add_default_created_updated_values.sql @@ -0,0 +1,15 @@ +ALTER TABLE IF EXISTS lists +ALTER COLUMN IF EXISTS created_at SET DEFAULT NOW(), +ALTER COLUMN IF EXISTS updated_at SET DEFAULT NOW(); + +ALTER TABLE IF EXISTS uris +ALTER COLUMN IF EXISTS created_at SET DEFAULT NOW(), +ALTER COLUMN IF EXISTS updated_at SET DEFAULT NOW(); + +ALTER TABLE IF EXISTS pages +ALTER COLUMN IF EXISTS created_at SET DEFAULT NOW(), +ALTER COLUMN IF EXISTS updated_at SET DEFAULT NOW(); + +ALTER TABLE IF EXISTS users +ALTER COLUMN IF EXISTS created_at SET DEFAULT NOW(), +ALTER COLUMN IF EXISTS updated_at SET DEFAULT NOW(); diff --git a/services/migrations/006_add_site_id_column_pages.sql b/services/migrations/006_add_site_id_column_pages.sql deleted file mode 100644 index 07debf1..0000000 --- a/services/migrations/006_add_site_id_column_pages.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE IF EXISTS pages -ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); diff --git a/services/migrations/007_add_site_id_column.sql b/services/migrations/007_add_site_id_column.sql new file mode 100644 index 0000000..2627379 --- /dev/null +++ b/services/migrations/007_add_site_id_column.sql @@ -0,0 +1,8 @@ +ALTER TABLE IF EXISTS lists +ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); + +ALTER TABLE IF EXISTS pages +ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); + +ALTER TABLE IF EXISTS uris +ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); diff --git a/services/migrations/007_add_site_id_column_lists.sql b/services/migrations/007_add_site_id_column_lists.sql deleted file mode 100644 index d7d3e61..0000000 --- a/services/migrations/007_add_site_id_column_lists.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE IF EXISTS lists -ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); diff --git a/services/migrations/008_add_site_id_column_uris.sql b/services/migrations/008_add_site_id_column_uris.sql deleted file mode 100644 index e04b8b7..0000000 --- a/services/migrations/008_add_site_id_column_uris.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE IF EXISTS uris -ADD COLUMN IF NOT EXISTS site_id VARCHAR(255); diff --git a/services/migrations/008_add_site_id_values_pages.sql b/services/migrations/008_add_site_id_values_pages.sql new file mode 100644 index 0000000..05ef51d --- /dev/null +++ b/services/migrations/008_add_site_id_values_pages.sql @@ -0,0 +1,7 @@ +UPDATE pages +SET site_id = subquery.site_slug +FROM (SELECT id, meta ->> 'siteSlug' as site_slug + FROM pages + WHERE meta IS NOT NULL + AND meta ->> 'siteSlug' IS NOT NULL) AS subquery +WHERE pages.id = subquery.id; diff --git a/services/migrations/009_add_site_id_values_uris.sql b/services/migrations/009_add_site_id_values_uris.sql new file mode 100644 index 0000000..8e28e10 --- /dev/null +++ b/services/migrations/009_add_site_id_values_uris.sql @@ -0,0 +1,7 @@ +UPDATE uris +SET site_id = subquery.site_slug +FROM (SELECT id, meta ->> 'siteSlug' as site_slug + FROM pages + WHERE meta IS NOT NULL + AND meta ->> 'siteSlug' IS NOT NULL) AS subquery +WHERE uris.data = subquery.id;