From 718f95cb5b13fbcad2efd9d2f1f29d8d7e973ed9 Mon Sep 17 00:00:00 2001 From: edu Date: Mon, 16 Sep 2019 16:35:27 +0200 Subject: [PATCH 1/3] move here logic for legacy messages --- lib/legacyMessages/helpers.js | 111 ++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 lib/legacyMessages/helpers.js diff --git a/lib/legacyMessages/helpers.js b/lib/legacyMessages/helpers.js new file mode 100644 index 00000000..8401b319 --- /dev/null +++ b/lib/legacyMessages/helpers.js @@ -0,0 +1,111 @@ +import { encryptMessage, decryptMessageLegacy } from 'pmcrypto'; +// import { fromUnixTime } from 'date-fns'; + +import { LABEL_LEGACY_MESSAGE, QUERY_LEGACY_MESSAGES_MAX_PAGESIZE } from '../constants'; +import { chunk } from '../helpers/array'; +import { wait } from '../helpers/promise'; +import { noop } from '../helpers/function'; + +const fromUnixTime = ({ Time = 0 } = {}) => new Date(Time * 1000); + +const LEGACY_MESSAGES_CHUNK_SIZE = 5; // how many messages we want to decrypt and encrypt simultaneously + +/** + * Given a list of legacy message IDs, fetch, decrypt, re-encrypt and send them to API + * @param {Array} messageIDs + * @param {Function} obj.getMessage + * @param {Function} obj.updateMessage + * @param {Function} obj.getPrivateKeys + * @param {Function} obj.getPublicKey + */ +const clearSome = async ( + messageIDs = [], + { signal, getMessage, updateMessage, getPrivateKeys, getPublicKey, onError = noop } +) => { + if (!messageIDs.length) { + return; + } + + return Promise.all( + messageIDs.map(async (ID) => { + try { + if (signal.aborted) { + return; + } + // Get message and private keys + const { data: message = {} } = await getMessage(ID); + const { Time, Body, AddressID } = message; + const privateKeys = getPrivateKeys(AddressID); + + if (signal.aborted) { + return; + } + // Decrypt message + const { data } = await decryptMessageLegacy({ + message: Body, + messageDate: fromUnixTime(Time), + privateKeys + }); + + if (signal.aborted) { + return; + } + // Re-encrypt message body. Use the primary key (first in the array) for re-encryption + const publicKeys = [getPublicKey(privateKeys[0])]; + const { data: newBody } = await encryptMessage({ + data, + publicKeys, + format: 'utf8', + compression: true + }); + + if (signal.aborted) { + return; + } + // Send re-encrypted message to API + return await updateMessage({ ID, Body: newBody }); + } catch (e) { + // Do nothing upon error + onError(e); + } + }) + ); +}; + +/** + * Fetch legacy messages, re-encrypt and send them to API + */ +export const clearAll = async ({ + signal, + queryMessages, + getMessage, + updateMessage, + getPrivateKeys, + getPublicKey, + onError +}) => { + const RELAX_TIME = 5 * 1000; // 5s + + if (signal.aborted) { + return; + } + const { data = {} } = await queryMessages({ + LabelID: [LABEL_LEGACY_MESSAGE], + Page: 0, + PageSize: QUERY_LEGACY_MESSAGES_MAX_PAGESIZE + }); + const { Total, Messages = [] } = data; + + if (Total === 0) { + return; + } + + // proceed in chunk of 10 messages, not to burn user's machine + const messageIDs = chunk(Messages.map(({ ID }) => ID), LEGACY_MESSAGES_CHUNK_SIZE); + for (let i = 0; i < messageIDs.length; i++) { + await clearSome(messageIDs[i], { getMessage, updateMessage, getPrivateKeys, getPublicKey, onError, signal }); + !signal.aborted && (await wait(RELAX_TIME)); + } + + return clearAll(); // updateMessage removes legacy label, so no need of updating the Page +}; From 008368c22f56d4f10526ce80229a5942c6d3ee3d Mon Sep 17 00:00:00 2001 From: edu Date: Thu, 19 Sep 2019 17:52:27 +0200 Subject: [PATCH 2/3] after tests --- lib/legacyMessages/helpers.js | 75 ++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 23 deletions(-) diff --git a/lib/legacyMessages/helpers.js b/lib/legacyMessages/helpers.js index 8401b319..32fc6926 100644 --- a/lib/legacyMessages/helpers.js +++ b/lib/legacyMessages/helpers.js @@ -1,45 +1,48 @@ import { encryptMessage, decryptMessageLegacy } from 'pmcrypto'; // import { fromUnixTime } from 'date-fns'; -import { LABEL_LEGACY_MESSAGE, QUERY_LEGACY_MESSAGES_MAX_PAGESIZE } from '../constants'; +import { LABEL_LEGACY_MESSAGE, QUERY_LEGACY_MESSAGES_MAX_PAGESIZE, UPDATE_MESSAGE_SUCESS_CODE } from '../constants'; import { chunk } from '../helpers/array'; import { wait } from '../helpers/promise'; import { noop } from '../helpers/function'; -const fromUnixTime = ({ Time = 0 } = {}) => new Date(Time * 1000); +const fromUnixTime = (Time = 0) => new Date(Time * 1000); const LEGACY_MESSAGES_CHUNK_SIZE = 5; // how many messages we want to decrypt and encrypt simultaneously /** * Given a list of legacy message IDs, fetch, decrypt, re-encrypt and send them to API - * @param {Array} messageIDs + * @param {Array} messageIDs List of IDs of legacy messages + * @param {AbortController.signal} signal { aborted: true/false } * @param {Function} obj.getMessage - * @param {Function} obj.updateMessage + * @param {Function} obj.updateBody * @param {Function} obj.getPrivateKeys - * @param {Function} obj.getPublicKey + * @param {Function} obj.onError */ -const clearSome = async ( - messageIDs = [], - { signal, getMessage, updateMessage, getPrivateKeys, getPublicKey, onError = noop } -) => { +const clearSome = async (messageIDs = [], { signal, getMessage, updateBody, getPrivateKeys, onError = noop }) => { if (!messageIDs.length) { return; } - return Promise.all( + let noneMigratedInBatch = true; + await Promise.all( messageIDs.map(async (ID) => { try { if (signal.aborted) { return; } + // Get message and private keys - const { data: message = {} } = await getMessage(ID); + const { + data: { Message: message = {} } + } = await getMessage(ID); const { Time, Body, AddressID } = message; - const privateKeys = getPrivateKeys(AddressID); + const privateKeys = await getPrivateKeys(AddressID); if (signal.aborted) { return; } + // Decrypt message const { data } = await decryptMessageLegacy({ message: Body, @@ -50,8 +53,9 @@ const clearSome = async ( if (signal.aborted) { return; } + // Re-encrypt message body. Use the primary key (first in the array) for re-encryption - const publicKeys = [getPublicKey(privateKeys[0])]; + const publicKeys = [privateKeys[0].toPublic()]; const { data: newBody } = await encryptMessage({ data, publicKeys, @@ -63,13 +67,19 @@ const clearSome = async ( return; } // Send re-encrypted message to API - return await updateMessage({ ID, Body: newBody }); + const { Code } = await updateBody({ ID, Body: newBody }); + if (Code === UPDATE_MESSAGE_SUCESS_CODE) { + noneMigratedInBatch = false; + } + return; } catch (e) { - // Do nothing upon error + e.message = `Error migrating legacy message with messageID ${ID}. ${e.message}`; onError(e); } }) ); + + return { noneMigratedInBatch }; }; /** @@ -77,11 +87,11 @@ const clearSome = async ( */ export const clearAll = async ({ signal, + page = 0, queryMessages, getMessage, - updateMessage, + updateBody, getPrivateKeys, - getPublicKey, onError }) => { const RELAX_TIME = 5 * 1000; // 5s @@ -89,23 +99,42 @@ export const clearAll = async ({ if (signal.aborted) { return; } + const { data = {} } = await queryMessages({ LabelID: [LABEL_LEGACY_MESSAGE], - Page: 0, + Page: page, PageSize: QUERY_LEGACY_MESSAGES_MAX_PAGESIZE }); - const { Total, Messages = [] } = data; + const { Messages = [] } = data; - if (Total === 0) { + if (!Messages.length) { return; } - // proceed in chunk of 10 messages, not to burn user's machine + // proceed in batches of messages, not to burn user's machine decrypting and re-encrypting const messageIDs = chunk(Messages.map(({ ID }) => ID), LEGACY_MESSAGES_CHUNK_SIZE); + let noneMigrated = true; for (let i = 0; i < messageIDs.length; i++) { - await clearSome(messageIDs[i], { getMessage, updateMessage, getPrivateKeys, getPublicKey, onError, signal }); + const { noneMigratedInBatch } = await clearSome(messageIDs[i], { + signal, + getMessage, + updateBody, + getPrivateKeys, + onError + }); + noneMigrated = noneMigrated && noneMigratedInBatch; !signal.aborted && (await wait(RELAX_TIME)); } - return clearAll(); // updateMessage removes legacy label, so no need of updating the Page + // updateBody removes legacy label only if update successful + // change page accordingly + return clearAll({ + signal, + page: page + noneMigrated, + queryMessages, + getMessage, + updateBody, + getPrivateKeys, + onError + }); }; From 19c0bfb9c964468f61a1d834a946c50ae50423fd Mon Sep 17 00:00:00 2001 From: edu Date: Thu, 19 Sep 2019 18:02:58 +0200 Subject: [PATCH 3/3] improve JSDoc --- lib/legacyMessages/helpers.js | 52 +++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/lib/legacyMessages/helpers.js b/lib/legacyMessages/helpers.js index 32fc6926..50754413 100644 --- a/lib/legacyMessages/helpers.js +++ b/lib/legacyMessages/helpers.js @@ -1,23 +1,29 @@ import { encryptMessage, decryptMessageLegacy } from 'pmcrypto'; // import { fromUnixTime } from 'date-fns'; -import { LABEL_LEGACY_MESSAGE, QUERY_LEGACY_MESSAGES_MAX_PAGESIZE, UPDATE_MESSAGE_SUCESS_CODE } from '../constants'; import { chunk } from '../helpers/array'; import { wait } from '../helpers/promise'; import { noop } from '../helpers/function'; const fromUnixTime = (Time = 0) => new Date(Time * 1000); - +const LABEL_LEGACY_MESSAGE = '11'; +const QUERY_LEGACY_MESSAGES_MAX_PAGESIZE = 150; +const UPDATE_BODY_SUCESS_CODE = 1000; const LEGACY_MESSAGES_CHUNK_SIZE = 5; // how many messages we want to decrypt and encrypt simultaneously +const RELAX_TIME = 5 * 1000; // 5s . Time to wait (for other operations) after a batch of legacy messages has been migrated /** * Given a list of legacy message IDs, fetch, decrypt, re-encrypt and send them to API * @param {Array} messageIDs List of IDs of legacy messages * @param {AbortController.signal} signal { aborted: true/false } - * @param {Function} obj.getMessage - * @param {Function} obj.updateBody - * @param {Function} obj.getPrivateKeys + * @param {Function} obj.getMessage Retrive message from database by ID + * @param {Function} obj.updateBody Replace body of a message identified by ID + * @param {Function} obj.getPrivateKeys Get privateKeys corresponding to an address ID * @param {Function} obj.onError + * + * @return {Object} { noneMigratedInBatch: false is some message from the batch is succesfully migrated. True otherwise } + * + * @dev The keys returned by getPrivateKeys must have the method toPublic() */ const clearSome = async (messageIDs = [], { signal, getMessage, updateBody, getPrivateKeys, onError = noop }) => { if (!messageIDs.length) { @@ -34,7 +40,7 @@ const clearSome = async (messageIDs = [], { signal, getMessage, updateBody, getP // Get message and private keys const { - data: { Message: message = {} } + data: { Message: message = {} }, } = await getMessage(ID); const { Time, Body, AddressID } = message; const privateKeys = await getPrivateKeys(AddressID); @@ -47,7 +53,7 @@ const clearSome = async (messageIDs = [], { signal, getMessage, updateBody, getP const { data } = await decryptMessageLegacy({ message: Body, messageDate: fromUnixTime(Time), - privateKeys + privateKeys, }); if (signal.aborted) { @@ -60,7 +66,7 @@ const clearSome = async (messageIDs = [], { signal, getMessage, updateBody, getP data, publicKeys, format: 'utf8', - compression: true + compression: true, }); if (signal.aborted) { @@ -68,7 +74,7 @@ const clearSome = async (messageIDs = [], { signal, getMessage, updateBody, getP } // Send re-encrypted message to API const { Code } = await updateBody({ ID, Body: newBody }); - if (Code === UPDATE_MESSAGE_SUCESS_CODE) { + if (Code === UPDATE_BODY_SUCESS_CODE) { noneMigratedInBatch = false; } return; @@ -84,6 +90,15 @@ const clearSome = async (messageIDs = [], { signal, getMessage, updateBody, getP /** * Fetch legacy messages, re-encrypt and send them to API + * @param {AbortController.signal} signal { aborted: true/false } + * @param {Number} page Legacy messages query parameter + * @param {Function} obj.queryMessages Retrive legacy messages from database + * @param {Function} obj.getMessage Retrive message from database by ID + * @param {Function} obj.updateBody Replace body of a message identified by ID + * @param {Function} obj.getPrivateKeys Get privateKeys corresponding to an address ID + * @param {Function} obj.onError + * + * @dev The keys returned by getPrivateKeys must have the method toPublic() */ export const clearAll = async ({ signal, @@ -92,10 +107,8 @@ export const clearAll = async ({ getMessage, updateBody, getPrivateKeys, - onError + onError, }) => { - const RELAX_TIME = 5 * 1000; // 5s - if (signal.aborted) { return; } @@ -103,7 +116,7 @@ export const clearAll = async ({ const { data = {} } = await queryMessages({ LabelID: [LABEL_LEGACY_MESSAGE], Page: page, - PageSize: QUERY_LEGACY_MESSAGES_MAX_PAGESIZE + PageSize: QUERY_LEGACY_MESSAGES_MAX_PAGESIZE, }); const { Messages = [] } = data; @@ -112,7 +125,10 @@ export const clearAll = async ({ } // proceed in batches of messages, not to burn user's machine decrypting and re-encrypting - const messageIDs = chunk(Messages.map(({ ID }) => ID), LEGACY_MESSAGES_CHUNK_SIZE); + const messageIDs = chunk( + Messages.map(({ ID }) => ID), + LEGACY_MESSAGES_CHUNK_SIZE + ); let noneMigrated = true; for (let i = 0; i < messageIDs.length; i++) { const { noneMigratedInBatch } = await clearSome(messageIDs[i], { @@ -120,10 +136,12 @@ export const clearAll = async ({ getMessage, updateBody, getPrivateKeys, - onError + onError, }); noneMigrated = noneMigrated && noneMigratedInBatch; - !signal.aborted && (await wait(RELAX_TIME)); + if (!signal.aborted) { + await wait(RELAX_TIME); + } } // updateBody removes legacy label only if update successful @@ -135,6 +153,6 @@ export const clearAll = async ({ getMessage, updateBody, getPrivateKeys, - onError + onError, }); };