diff --git a/postgres/client.js b/postgres/client.js index 3d55edb..ca0a5df 100644 --- a/postgres/client.js +++ b/postgres/client.js @@ -5,7 +5,7 @@ const { POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST, POSTGRES_PORT, POSTGRES { parseOrNot, wrapInObject, decode } = require('../services/utils'), { findSchemaAndTable, wrapJSONStringInObject } = require('../services/utils'), knexLib = require('knex'), - { isList, isUri } = require('clayutils'), + { isList, isUri, isPage } = require('clayutils'), TransformStream = require('../services/list-transform-stream'), META_PUT_PATCH_FN = patch('meta'); var knex, log = require('../services/log').setup({ file: __filename }); @@ -272,16 +272,52 @@ function getMeta(key) { * @param {Object} value [description] * @return {Promise} [description] */ +/* eslint complexity: 0 */ function putMeta(key, value) { const { schema, table } = findSchemaAndTable(key), - map = columnToValueMap('id', key); + map = columnToValueMap('id', key), + parsedValue = parseOrNot(value); // add meta column to map - columnToValueMap('meta', parseOrNot(value), map); + columnToValueMap('meta', parsedValue, map); + + if (isPage(key)) { + // If value has first publish time, add it to the map + if (parsedValue.firstPublishTime) + columnToValueMap('published_at', parsedValue.firstPublishTime, map); + + // If value has publish time, add it as republished date + if (parsedValue.publishTime) + columnToValueMap('republished_at', parsedValue.publishTime, map); + + // If we have history data, then find the unpublish and archive events + if (parsedValue.history && parsedValue.history.length) { + const latestUnpublish = getLatestActionByName(parsedValue.history, 'unpublish'), + latestArchived = getLatestActionByName(parsedValue.history, 'archive'); + + if (latestUnpublish.timestamp) + columnToValueMap('unpublished_at', latestUnpublish.timestamp, map); + + if (latestArchived.timestamp) + columnToValueMap('archived_at', latestArchived.timestamp, map); + } + } return onConflictPut(map, schema, table).then(() => map.meta); } +/** + * Gets the latest entry in the history based on the action + * @param {Object[]} history + * @param {String} action + * @returns {Object} + */ +function getLatestActionByName(history, action) { + return history + .filter(event => event.action === action) + .pop() || {}; +} + /** * Creates a table with the name that's * passed into the function. Table has diff --git a/postgres/client.test.js b/postgres/client.test.js index 0ce693b..d41a1e7 100644 --- a/postgres/client.test.js +++ b/postgres/client.test.js @@ -605,6 +605,57 @@ describe('postgres/client', () => { expect(data).toEqual(data); }); }); + + test('if uri is page, insert publication events dates', () => { + const key = 'nymag.com/_pages/sample-page', + meta = { + someText: '', + someOtherText: '', + history: [{ + action: 'unpublish', + timestamp: '2019-06-14' + }, { + action: 'archive', + timestamp: '2019-06-14' + }] + }; + + return client.putMeta(key, meta).then((data) => { + expect(data).toEqual(meta); + }); + }); + + test('if uri is page and does not have history, insert without publication dates', () => { + const key = 'nymag.com/_pages/sample-page', + meta = { + someText: '', + someOtherText: '', + firstPublishTime: '2019-06-14', + publishTime: '2019-06-14' + }; + + return client.putMeta(key, meta).then((data) => { + expect(data).toEqual(meta); + }); + }); + + test('if uri is page, but history does not have unpublish/archive events, insert without those dates', () => { + const key = 'nymag.com/_pages/sample-page', + meta = { + someText: '', + someOtherText: '', + firstPublishTime: '2019-06-14', + publishTime: '2019-06-14', + history: [{ + action: 'create', + timestamp: '2019-06-14' + }] + }; + + return client.putMeta(key, meta).then((data) => { + expect(data).toEqual(meta); + }); + }); }); describe('batch', () => { diff --git a/postgres/index.js b/postgres/index.js index 6b7322b..9dce850 100644 --- a/postgres/index.js +++ b/postgres/index.js @@ -42,6 +42,7 @@ function createTables() { return bluebird.all(getComponents().map(component => client.createTable(`components.${component}`))) .then(() => bluebird.all(getLayouts().map(layout => client.createTableWithMeta(`layouts.${layout}`)))) .then(() => client.createTableWithMeta('pages')) + .then(() => client.raw('CREATE TABLE IF NOT EXISTS ?? ( id TEXT PRIMARY KEY NOT NULL, data TEXT NOT NULL, url TEXT );', ['uris'])) .then(() => createRemainingTables()); } @@ -59,24 +60,22 @@ function setup(testPostgresHost) { } return client.connect() - .then(() => { - return migrate( - { - database: POSTGRES_DB, - user: POSTGRES_USER, - password: POSTGRES_PASSWORD, - host: postgresHost, - port: POSTGRES_PORT - }, - path.join(__dirname, '../services/migrations') - ); - }) - .then(() => { - log('info', 'Migrations Complete'); - }) - .then(() => createTables()) + .then(() => client.createSchema('components')) + .then(() => client.createSchema('layouts')) + .then(createTables) + .then(() => migrate( + { + database: POSTGRES_DB, + user: POSTGRES_USER, + password: POSTGRES_PASSWORD, + host: postgresHost, + port: POSTGRES_PORT + }, + path.join(__dirname, '../services/migrations') + )) + .then(() => log('info', 'Migrations Complete')) .then(() => ({ server: `${postgresHost}:${POSTGRES_PORT}` })) - .catch(logGenericError); + .catch(logGenericError(__filename)); } module.exports.setup = setup; diff --git a/services/migrations/001_add_publish_event_columns.sql b/services/migrations/001_add_publish_event_columns.sql new file mode 100644 index 0000000..07e0f8d --- /dev/null +++ b/services/migrations/001_add_publish_event_columns.sql @@ -0,0 +1,5 @@ +ALTER TABLE IF EXISTS pages +ADD COLUMN IF NOT EXISTS published_at TIMESTAMPTZ, +ADD COLUMN IF NOT EXISTS unpublished_at TIMESTAMPTZ, +ADD COLUMN IF NOT EXISTS republished_at TIMESTAMPTZ, +ADD COLUMN IF NOT EXISTS archived_at TIMESTAMPTZ; diff --git a/services/migrations/001_create_components_schema.sql b/services/migrations/001_create_components_schema.sql deleted file mode 100644 index 042d10e..0000000 --- a/services/migrations/001_create_components_schema.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE SCHEMA IF NOT EXISTS components; diff --git a/services/migrations/002_create_layouts_schema.sql b/services/migrations/002_create_layouts_schema.sql deleted file mode 100644 index 8a3e1dc..0000000 --- a/services/migrations/002_create_layouts_schema.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE SCHEMA IF NOT EXISTS layouts; diff --git a/services/migrations/002_set_unpublished_at_values.sql b/services/migrations/002_set_unpublished_at_values.sql new file mode 100644 index 0000000..8191f81 --- /dev/null +++ b/services/migrations/002_set_unpublished_at_values.sql @@ -0,0 +1,11 @@ +UPDATE pages +SET unpublished_at = subquery.unpublish_date +FROM (SELECT DISTINCT ON (id) id, + events ->> 'action' AS action, + TO_TIMESTAMP(events ->> 'timestamp', 'YYYY-MM-DD HH24:MI:SSZ') AS unpublish_date + FROM pages AS p, + JSONB_ARRAY_ELEMENTS(p.meta -> 'history') AS events + WHERE meta -> 'history' IS NOT NULL + AND events ->> 'action' = 'unpublish' + ORDER BY id, unpublish_date DESC) AS subquery +WHERE pages.id = subquery.id; diff --git a/services/migrations/003_create_pages_table.sql b/services/migrations/003_create_pages_table.sql deleted file mode 100644 index 8bb3792..0000000 --- a/services/migrations/003_create_pages_table.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE TABLE IF NOT EXISTS "pages" ( id TEXT PRIMARY KEY NOT NULL, data JSONB, meta JSONB ); diff --git a/services/migrations/003_set_archived_at_values.sql b/services/migrations/003_set_archived_at_values.sql new file mode 100644 index 0000000..6862a91 --- /dev/null +++ b/services/migrations/003_set_archived_at_values.sql @@ -0,0 +1,11 @@ +UPDATE pages +SET archived_at = subquery.archive_date +FROM (SELECT DISTINCT ON (id) id, + events ->> 'action' AS action, + TO_TIMESTAMP(events ->> 'timestamp', 'YYYY-MM-DD HH24:MI:SSZ') AS archive_date + FROM pages AS p, + JSONB_ARRAY_ELEMENTS(p.meta -> 'history') AS events + WHERE meta -> 'history' IS NOT NULL + AND events ->> 'action' = 'archive' + ORDER BY id, archive_date DESC) AS subquery +WHERE pages.id = subquery.id; diff --git a/services/migrations/004_create_uris_table.sql b/services/migrations/004_create_uris_table.sql deleted file mode 100644 index 2d74761..0000000 --- a/services/migrations/004_create_uris_table.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE TABLE IF NOT EXISTS "uris" ( id TEXT PRIMARY KEY NOT NULL, data TEXT NOT NULL, url TEXT ); diff --git a/services/migrations/004_set_republished_at_values.sql b/services/migrations/004_set_republished_at_values.sql new file mode 100644 index 0000000..e8ef980 --- /dev/null +++ b/services/migrations/004_set_republished_at_values.sql @@ -0,0 +1,7 @@ +UPDATE pages +SET republished_at = subquery.republish_time +FROM (SELECT id, TO_TIMESTAMP(meta ->> 'publishTime', 'YYYY-MM-DD HH24:MI:SSZ') AS republish_time + FROM pages + WHERE meta IS NOT NULL + AND meta ->> 'publishTime' IS NOT NULL) AS subquery +WHERE pages.id = subquery.id; diff --git a/services/migrations/005_set_published_at_values.sql b/services/migrations/005_set_published_at_values.sql new file mode 100644 index 0000000..4c24cc8 --- /dev/null +++ b/services/migrations/005_set_published_at_values.sql @@ -0,0 +1,7 @@ +UPDATE pages +SET published_at = subquery.first_publish_time +FROM (SELECT id, TO_TIMESTAMP(meta ->> 'firstPublishTime', 'YYYY-MM-DD HH24:MI:SSZ') as first_publish_time + FROM pages + WHERE meta IS NOT NULL + AND meta ->> 'firstPublishTime' IS NOT NULL) AS subquery +WHERE pages.id = subquery.id;