From d9d64089d552354fd477f63c476fb90899fd8291 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 1 Oct 2024 06:15:05 +0100 Subject: [PATCH 001/166] Expose LiveObjects as a plugin Base code, tests and build setup for new LiveObjects plugin. Adds a new `.liveObjects` property for RealtimeChannel. Plugin setup is based on Web Push plugin PR [1], and CDN setup for Push plugin PR [2]. Resolves DTP-947 [1] https://github.com/ably/ably-js/pull/1775 [2] https://github.com/ably/ably-js/pull/1861 --- Gruntfile.js | 26 +++++++++- README.md | 35 +++++++++++++ ably.d.ts | 14 ++++++ grunt/esbuild/build.js | 25 ++++++++++ liveobjects.d.ts | 28 +++++++++++ package.json | 10 +++- scripts/cdn_deploy.js | 2 +- scripts/moduleReport.ts | 31 +++++++++--- src/common/lib/client/modularplugins.ts | 2 + src/common/lib/client/realtimechannel.ts | 13 +++++ src/plugins/index.d.ts | 2 + src/plugins/liveobjects/index.ts | 7 +++ src/plugins/liveobjects/liveobjects.ts | 12 +++++ test/common/globals/named_dependencies.js | 4 ++ test/realtime/live_objects.test.js | 60 +++++++++++++++++++++++ test/support/browser_file_list.js | 1 + 16 files changed, 262 insertions(+), 10 deletions(-) create mode 100644 liveobjects.d.ts create mode 100644 src/plugins/liveobjects/index.ts create mode 100644 src/plugins/liveobjects/liveobjects.ts create mode 100644 test/realtime/live_objects.test.js diff --git a/Gruntfile.js b/Gruntfile.js index 3bd1b0c233..fdace117dd 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -73,7 +73,14 @@ module.exports = function (grunt) { }); }); - grunt.registerTask('build', ['checkGitSubmodules', 'webpack:all', 'build:browser', 'build:node', 'build:push']); + grunt.registerTask('build', [ + 'checkGitSubmodules', + 'webpack:all', + 'build:browser', + 'build:node', + 'build:push', + 'build:liveobjects', + ]); grunt.registerTask('all', ['build', 'requirejs']); @@ -138,9 +145,26 @@ module.exports = function (grunt) { }); }); + grunt.registerTask('build:liveobjects', function () { + var done = this.async(); + + Promise.all([ + esbuild.build(esbuildConfig.liveObjectsPluginConfig), + esbuild.build(esbuildConfig.liveObjectsPluginCdnConfig), + esbuild.build(esbuildConfig.minifiedLiveObjectsPluginCdnConfig), + ]) + .then(() => { + done(true); + }) + .catch((err) => { + done(err); + }); + }); + grunt.registerTask('test:webserver', 'Launch the Mocha test web server on http://localhost:3000/', [ 'build:browser', 'build:push', + 'build:liveobjects', 'checkGitSubmodules', 'mocha:webserver', ]); diff --git a/README.md b/README.md index 0f7c27796b..78a2b81a26 100644 --- a/README.md +++ b/README.md @@ -586,6 +586,41 @@ The Push plugin is developed as part of the Ably client library, so it is availa For more information on publishing push notifcations over Ably, see the [Ably push documentation](https://ably.com/docs/push). +### Live Objects functionality + +Live Objects functionality is supported for Realtime clients via the LiveObjects plugin. In order to use Live Objects, you must pass in the plugin via client options. + +```javascript +import * as Ably from 'ably'; +import LiveObjects from 'ably/liveobjects'; + +const client = new Ably.Realtime({ + ...options, + plugins: { LiveObjects }, +}); +``` + +LiveObjects plugin also works with the [Modular variant](#modular-tree-shakable-variant) of the library. + +Alternatively, you can load the LiveObjects plugin directly in your HTML using `script` tag (in case you can't use a package manager): + +```html + +``` + +When loaded this way, the LiveObjects plugin will be available on the global object via the `AblyLiveObjectsPlugin` property, so you will need to pass it to the Ably instance as follows: + +```javascript +const client = new Ably.Realtime({ + ...options, + plugins: { LiveObjects: AblyLiveObjectsPlugin }, +}); +``` + +The LiveObjects plugin is developed as part of the Ably client library, so it is available for the same versions as the Ably client library itself. It also means that it follows the same semantic versioning rules as they were defined for [the Ably client library](#for-browsers). For example, to lock into a major or minor version of the LiveObjects plugin, you can specify a specific version number such as https://cdn.ably.com/lib/liveobjects.umd.min-2.js for all v2._ versions, or https://cdn.ably.com/lib/liveobjects.umd.min-2.4.js for all v2.4._ versions, or you can lock into a single release with https://cdn.ably.com/lib/liveobjects.umd.min-2.4.0.js. Note you can load the non-minified version by omitting `.min` from the URL such as https://cdn.ably.com/lib/liveobjects.umd-2.js. + +For more information about Live Objects product, see the [Ably Live Objects documentation](https://ably.com/docs/products/liveobjects). + ## Delta Plugin From version 1.2 this client library supports subscription to a stream of Vcdiff formatted delta messages from the Ably service. For certain applications this can bring significant data efficiency savings. diff --git a/ably.d.ts b/ably.d.ts index b8e85c6a4b..18b7ee7208 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -623,6 +623,11 @@ export interface CorePlugins { * A plugin which allows the client to be the target of push notifications. */ Push?: unknown; + + /** + * A plugin which allows the client to use LiveObjects functionality at {@link RealtimeChannel.liveObjects}. + */ + LiveObjects?: unknown; } /** @@ -2010,6 +2015,11 @@ export declare interface PushChannel { listSubscriptions(params?: Record): Promise>; } +/** + * Enables the LiveObjects state to be subscribed to for a channel. + */ +export declare interface LiveObjects {} + /** * Enables messages to be published and historic messages to be retrieved for a channel. */ @@ -2139,6 +2149,10 @@ export declare interface RealtimeChannel extends EventEmitter { return output; } -async function calculatePushPluginSize(): Promise { +async function calculatePluginSize(options: { path: string; description: string }): Promise { const output: Output = { tableRows: [], errors: [] }; - const pushPluginBundleInfo = getBundleInfo('./build/push.js'); + const pluginBundleInfo = getBundleInfo(options.path); const sizes = { - rawByteSize: pushPluginBundleInfo.byteSize, - gzipEncodedByteSize: (await promisify(gzip)(pushPluginBundleInfo.code)).byteLength, + rawByteSize: pluginBundleInfo.byteSize, + gzipEncodedByteSize: (await promisify(gzip)(pluginBundleInfo.code)).byteLength, }; output.tableRows.push({ - description: 'Push', + description: options.description, sizes: sizes, }); return output; } +async function calculatePushPluginSize(): Promise { + return calculatePluginSize({ path: './build/push.js', description: 'Push' }); +} + +async function calculateLiveObjectsPluginSize(): Promise { + return calculatePluginSize({ path: './build/liveobjects.js', description: 'LiveObjects' }); +} + async function calculateAndCheckMinimalUsefulRealtimeBundleSize(): Promise { const output: Output = { tableRows: [], errors: [] }; @@ -296,6 +304,15 @@ async function checkPushPluginFiles() { return checkBundleFiles(pushPluginBundleInfo, allowedFiles, 100); } +async function checkLiveObjectsPluginFiles() { + const pluginBundleInfo = getBundleInfo('./build/liveobjects.js'); + + // These are the files that are allowed to contribute >= `threshold` bytes to the LiveObjects bundle. + const allowedFiles = new Set(['src/plugins/liveobjects/index.ts']); + + return checkBundleFiles(pluginBundleInfo, allowedFiles, 100); +} + async function checkBundleFiles(bundleInfo: BundleInfo, allowedFiles: Set, thresholdBytes: number) { const exploreResult = await runSourceMapExplorer(bundleInfo); @@ -347,6 +364,7 @@ async function checkBundleFiles(bundleInfo: BundleInfo, allowedFiles: Set ({ tableRows: [...accum.tableRows, ...current.tableRows], @@ -355,6 +373,7 @@ async function checkBundleFiles(bundleInfo: BundleInfo, allowedFiles: Set { + /** @nospec */ + it("throws an error when attempting to access the channel's `liveObjects` property", async function () { + const helper = this.test.helper; + const client = helper.AblyRealtime({ autoConnect: false }); + const channel = client.channels.get('channel'); + expect(() => channel.liveObjects).to.throw('LiveObjects plugin not provided'); + }); + }); + + describe('Realtime with LiveObjects plugin', () => { + /** @nospec */ + it("returns LiveObjects instance when accessing channel's `liveObjects` property", async function () { + const helper = this.test.helper; + const client = LiveObjectsRealtime(helper, { autoConnect: false }); + const channel = client.channels.get('channel'); + expect(channel.liveObjects.constructor.name).to.equal('LiveObjects'); + }); + }); + }); +}); diff --git a/test/support/browser_file_list.js b/test/support/browser_file_list.js index 80d5d8d8b1..d49cbee9e7 100644 --- a/test/support/browser_file_list.js +++ b/test/support/browser_file_list.js @@ -39,6 +39,7 @@ window.__testFiles__.files = { 'test/realtime/failure.test.js': true, 'test/realtime/history.test.js': true, 'test/realtime/init.test.js': true, + 'test/realtime/live_objects.test.js': true, 'test/realtime/message.test.js': true, 'test/realtime/presence.test.js': true, 'test/realtime/reauth.test.js': true, From fac8a6412351433ffa1bf70d69106205118dd2f6 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 4 Oct 2024 05:50:00 +0100 Subject: [PATCH 002/166] Move `monitorConnectionThenCloseAndFinish` to shared helper --- test/browser/modular.test.js | 394 ++++++++++++--------------- test/common/modules/shared_helper.js | 8 + test/realtime/live_objects.test.js | 8 - 3 files changed, 175 insertions(+), 235 deletions(-) diff --git a/test/browser/modular.test.js b/test/browser/modular.test.js index 739b1becbe..863db33482 100644 --- a/test/browser/modular.test.js +++ b/test/browser/modular.test.js @@ -32,14 +32,6 @@ function registerAblyModularTests(Helper) { }); }; - async function monitorConnectionThenCloseAndFinish(helper, action, realtime, states) { - try { - await helper.monitorConnectionAsync(action, realtime, states); - } finally { - await helper.closeAndFinishAsync(realtime); - } - } - before(function (done) { const helper = Helper.forHook(this); helper.setupApp(done); @@ -202,25 +194,21 @@ function registerAblyModularTests(Helper) { this.test.helper.ablyClientOptions({ plugins: { WebSocketTransport, FetchRequest } }), ); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - const channel = client.channels.get('channel'); - await channel.attach(); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get('channel'); + await channel.attach(); - const recievedMessagePromise = new Promise((resolve) => { - channel.subscribe((message) => { - resolve(message); - }); + const recievedMessagePromise = new Promise((resolve) => { + channel.subscribe((message) => { + resolve(message); }); + }); - await channel.publish({ data: { foo: 'bar' } }); + await channel.publish({ data: { foo: 'bar' } }); - const receivedMessage = await recievedMessagePromise; - expect(receivedMessage.data).to.eql({ foo: 'bar' }); - }, - client, - ); + const receivedMessage = await recievedMessagePromise; + expect(receivedMessage.data).to.eql({ foo: 'bar' }); + }, client); }); /** @nospec */ @@ -463,48 +451,44 @@ function registerAblyModularTests(Helper) { const rxClient = new BaseRealtime({ ...clientOptions, plugins: { WebSocketTransport, FetchRequest } }); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - const rxChannel = rxClient.channels.get('channel'); - await rxChannel.attach(); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const rxChannel = rxClient.channels.get('channel'); + await rxChannel.attach(); - const rxMessagePromise = new Promise((resolve, _) => rxChannel.subscribe((message) => resolve(message))); + const rxMessagePromise = new Promise((resolve, _) => rxChannel.subscribe((message) => resolve(message))); - const encryptionChannelOptions = { cipher: { key } }; + const encryptionChannelOptions = { cipher: { key } }; - const txMessage = { name: 'message', data: 'data' }; - const txClient = new clientClassConfig.clientClass({ - ...clientOptions, - plugins: { - ...clientClassConfig.additionalPlugins, - FetchRequest, - Crypto, - }, - }); + const txMessage = { name: 'message', data: 'data' }; + const txClient = new clientClassConfig.clientClass({ + ...clientOptions, + plugins: { + ...clientClassConfig.additionalPlugins, + FetchRequest, + Crypto, + }, + }); - await ( - clientClassConfig.isRealtime ? monitorConnectionThenCloseAndFinish : async (helper, op) => await op() - )( - helper, - async () => { - const txChannel = txClient.channels.get('channel', encryptionChannelOptions); - await txChannel.publish(txMessage); + const action = async () => { + const txChannel = txClient.channels.get('channel', encryptionChannelOptions); + await txChannel.publish(txMessage); - const rxMessage = await rxMessagePromise; + const rxMessage = await rxMessagePromise; - // Verify that the message was published with encryption - expect(rxMessage.encoding).to.equal('utf-8/cipher+aes-256-cbc'); + // Verify that the message was published with encryption + expect(rxMessage.encoding).to.equal('utf-8/cipher+aes-256-cbc'); - // Verify that the message was correctly encrypted - const rxMessageDecrypted = await decodeEncryptedMessage(rxMessage, encryptionChannelOptions); - helper.testMessageEquality(rxMessageDecrypted, txMessage); - }, - txClient, - ); - }, - rxClient, - ); + // Verify that the message was correctly encrypted + const rxMessageDecrypted = await decodeEncryptedMessage(rxMessage, encryptionChannelOptions); + helper.testMessageEquality(rxMessageDecrypted, txMessage); + }; + + if (clientClassConfig.isRealtime) { + await helper.monitorConnectionThenCloseAndFinish(action, txClient); + } else { + await action(); + } + }, rxClient); } for (const clientClassConfig of [ @@ -585,13 +569,9 @@ function registerAblyModularTests(Helper) { }), ); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - await testRealtimeUsesFormat(client, 'json'); - }, - client, - ); + await helper.monitorConnectionThenCloseAndFinish(async () => { + await testRealtimeUsesFormat(client, 'json'); + }, client); }); }); }); @@ -629,13 +609,9 @@ function registerAblyModularTests(Helper) { }), ); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - await testRealtimeUsesFormat(client, 'msgpack'); - }, - client, - ); + await helper.monitorConnectionThenCloseAndFinish(async () => { + await testRealtimeUsesFormat(client, 'msgpack'); + }, client); }); }); }); @@ -649,15 +625,11 @@ function registerAblyModularTests(Helper) { const helper = this.test.helper; const client = new BaseRealtime(helper.ablyClientOptions({ plugins: { WebSocketTransport, FetchRequest } })); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - const channel = client.channels.get('channel'); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get('channel'); - expect(() => channel.presence).to.throw('RealtimePresence plugin not provided'); - }, - client, - ); + expect(() => channel.presence).to.throw('RealtimePresence plugin not provided'); + }, client); }); /** @nospec */ @@ -667,43 +639,35 @@ function registerAblyModularTests(Helper) { helper.ablyClientOptions({ plugins: { WebSocketTransport, FetchRequest } }), ); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - const rxChannel = rxClient.channels.get('channel'); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const rxChannel = rxClient.channels.get('channel'); - await rxChannel.attach(); + await rxChannel.attach(); - const receivedMessagePromise = new Promise((resolve) => rxChannel.subscribe(resolve)); + const receivedMessagePromise = new Promise((resolve) => rxChannel.subscribe(resolve)); - const txClient = new BaseRealtime( - this.test.helper.ablyClientOptions({ - clientId: Helper.randomString(), - plugins: { - WebSocketTransport, - FetchRequest, - RealtimePresence, - }, - }), - ); + const txClient = new BaseRealtime( + this.test.helper.ablyClientOptions({ + clientId: Helper.randomString(), + plugins: { + WebSocketTransport, + FetchRequest, + RealtimePresence, + }, + }), + ); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - const txChannel = txClient.channels.get('channel'); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const txChannel = txClient.channels.get('channel'); - await txChannel.publish('message', 'body'); - await txChannel.presence.enter(); + await txChannel.publish('message', 'body'); + await txChannel.presence.enter(); - // The idea being here that in order for receivedMessagePromise to resolve, rxClient must have first processed the PRESENCE ProtocolMessage that resulted from txChannel.presence.enter() + // The idea being here that in order for receivedMessagePromise to resolve, rxClient must have first processed the PRESENCE ProtocolMessage that resulted from txChannel.presence.enter() - await receivedMessagePromise; - }, - txClient, - ); - }, - rxClient, - ); + await receivedMessagePromise; + }, txClient); + }, rxClient); }); }); @@ -727,41 +691,33 @@ function registerAblyModularTests(Helper) { ); const rxChannel = rxClient.channels.get('channel'); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - const txClientId = Helper.randomString(); - const txClient = new BaseRealtime( - this.test.helper.ablyClientOptions({ - clientId: txClientId, - plugins: { - WebSocketTransport, - FetchRequest, - RealtimePresence, - }, - }), - ); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const txClientId = Helper.randomString(); + const txClient = new BaseRealtime( + this.test.helper.ablyClientOptions({ + clientId: txClientId, + plugins: { + WebSocketTransport, + FetchRequest, + RealtimePresence, + }, + }), + ); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - const txChannel = txClient.channels.get('channel'); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const txChannel = txClient.channels.get('channel'); - let resolveRxPresenceMessagePromise; - const rxPresenceMessagePromise = new Promise((resolve, reject) => { - resolveRxPresenceMessagePromise = resolve; - }); - await rxChannel.presence.subscribe('enter', resolveRxPresenceMessagePromise); - await txChannel.presence.enter(); + let resolveRxPresenceMessagePromise; + const rxPresenceMessagePromise = new Promise((resolve, reject) => { + resolveRxPresenceMessagePromise = resolve; + }); + await rxChannel.presence.subscribe('enter', resolveRxPresenceMessagePromise); + await txChannel.presence.enter(); - const rxPresenceMessage = await rxPresenceMessagePromise; - expect(rxPresenceMessage.clientId).to.equal(txClientId); - }, - txClient, - ); - }, - rxClient, - ); + const rxPresenceMessage = await rxPresenceMessagePromise; + expect(rxPresenceMessage.clientId).to.equal(txClientId); + }, txClient); + }, rxClient); }); }); }); @@ -855,26 +811,22 @@ function registerAblyModularTests(Helper) { }), ); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - let firstTransportCandidate; - const connectionManager = realtime.connection.connectionManager; - const originalTryATransport = connectionManager.tryATransport; - realtime.connection.connectionManager.tryATransport = (transportParams, candidate, callback) => { - if (!firstTransportCandidate) { - firstTransportCandidate = candidate; - } - originalTryATransport.bind(connectionManager)(transportParams, candidate, callback); - }; - - realtime.connect(); - - await realtime.connection.once('connected'); - expect(firstTransportCandidate).to.equal(scenario.transportName); - }, - realtime, - ); + await helper.monitorConnectionThenCloseAndFinish(async () => { + let firstTransportCandidate; + const connectionManager = realtime.connection.connectionManager; + const originalTryATransport = connectionManager.tryATransport; + realtime.connection.connectionManager.tryATransport = (transportParams, candidate, callback) => { + if (!firstTransportCandidate) { + firstTransportCandidate = candidate; + } + originalTryATransport.bind(connectionManager)(transportParams, candidate, callback); + }; + + realtime.connect(); + + await realtime.connection.once('connected'); + expect(firstTransportCandidate).to.equal(scenario.transportName); + }, realtime); }); }); } @@ -914,21 +866,17 @@ function registerAblyModularTests(Helper) { helper.ablyClientOptions({ plugins: { WebSocketTransport, FetchRequest } }), ); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - const channel = realtime.channels.get('channel'); - await channel.attach(); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = realtime.channels.get('channel'); + await channel.attach(); - const subscribeReceivedMessagePromise = new Promise((resolve) => channel.subscribe(resolve)); + const subscribeReceivedMessagePromise = new Promise((resolve) => channel.subscribe(resolve)); - await channel.publish('message', 'body'); + await channel.publish('message', 'body'); - const subscribeReceivedMessage = await subscribeReceivedMessagePromise; - expect(subscribeReceivedMessage.data).to.equal('body'); - }, - realtime, - ); + const subscribeReceivedMessage = await subscribeReceivedMessagePromise; + expect(subscribeReceivedMessage.data).to.equal('body'); + }, realtime); }); /** @nospec */ @@ -938,23 +886,19 @@ function registerAblyModularTests(Helper) { helper.ablyClientOptions({ plugins: { WebSocketTransport, FetchRequest } }), ); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - const channel = realtime.channels.get('channel'); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = realtime.channels.get('channel'); - let thrownError = null; - try { - await channel.subscribe({ clientId: 'someClientId' }, () => {}); - } catch (error) { - thrownError = error; - } + let thrownError = null; + try { + await channel.subscribe({ clientId: 'someClientId' }, () => {}); + } catch (error) { + thrownError = error; + } - expect(thrownError).not.to.be.null; - expect(thrownError.message).to.equal('MessageInteractions plugin not provided'); - }, - realtime, - ); + expect(thrownError).not.to.be.null; + expect(thrownError.message).to.equal('MessageInteractions plugin not provided'); + }, realtime); }); }); @@ -975,56 +919,52 @@ function registerAblyModularTests(Helper) { }), ); - await monitorConnectionThenCloseAndFinish( - helper, - async () => { - const channel = realtime.channels.get('channel'); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = realtime.channels.get('channel'); - await channel.attach(); + await channel.attach(); - // Test `subscribe` with a filter: send two messages with different clientIds, and check that unfiltered subscription receives both messages but clientId-filtered subscription only receives the matching one. - const messageFilter = { clientId: 'someClientId' }; // note that `unsubscribe` compares filter by reference, I found that a bit surprising + // Test `subscribe` with a filter: send two messages with different clientIds, and check that unfiltered subscription receives both messages but clientId-filtered subscription only receives the matching one. + const messageFilter = { clientId: 'someClientId' }; // note that `unsubscribe` compares filter by reference, I found that a bit surprising - const filteredSubscriptionReceivedMessages = []; - channel.subscribe(messageFilter, (message) => { - filteredSubscriptionReceivedMessages.push(message); - }); + const filteredSubscriptionReceivedMessages = []; + channel.subscribe(messageFilter, (message) => { + filteredSubscriptionReceivedMessages.push(message); + }); - const unfilteredSubscriptionReceivedFirstTwoMessagesPromise = new Promise((resolve) => { - const receivedMessages = []; - channel.subscribe(function listener(message) { - receivedMessages.push(message); - if (receivedMessages.length === 2) { - channel.unsubscribe(listener); - resolve(); - } - }); + const unfilteredSubscriptionReceivedFirstTwoMessagesPromise = new Promise((resolve) => { + const receivedMessages = []; + channel.subscribe(function listener(message) { + receivedMessages.push(message); + if (receivedMessages.length === 2) { + channel.unsubscribe(listener); + resolve(); + } }); + }); - await channel.publish(await decodeMessage({ clientId: 'someClientId' })); - await channel.publish(await decodeMessage({ clientId: 'someOtherClientId' })); - await unfilteredSubscriptionReceivedFirstTwoMessagesPromise; + await channel.publish(await decodeMessage({ clientId: 'someClientId' })); + await channel.publish(await decodeMessage({ clientId: 'someOtherClientId' })); + await unfilteredSubscriptionReceivedFirstTwoMessagesPromise; - expect(filteredSubscriptionReceivedMessages.length).to.equal(1); - expect(filteredSubscriptionReceivedMessages[0].clientId).to.equal('someClientId'); + expect(filteredSubscriptionReceivedMessages.length).to.equal(1); + expect(filteredSubscriptionReceivedMessages[0].clientId).to.equal('someClientId'); - // Test `unsubscribe` with a filter: call `unsubscribe` with the clientId filter, publish a message matching the filter, check that only the unfiltered listener recieves it - channel.unsubscribe(messageFilter); + // Test `unsubscribe` with a filter: call `unsubscribe` with the clientId filter, publish a message matching the filter, check that only the unfiltered listener recieves it + channel.unsubscribe(messageFilter); - const unfilteredSubscriptionReceivedNextMessagePromise = new Promise((resolve) => { - channel.subscribe(function listener() { - channel.unsubscribe(listener); - resolve(); - }); + const unfilteredSubscriptionReceivedNextMessagePromise = new Promise((resolve) => { + channel.subscribe(function listener() { + channel.unsubscribe(listener); + resolve(); }); + }); - await channel.publish(await decodeMessage({ clientId: 'someClientId' })); - await unfilteredSubscriptionReceivedNextMessagePromise; + await channel.publish(await decodeMessage({ clientId: 'someClientId' })); + await unfilteredSubscriptionReceivedNextMessagePromise; - expect(filteredSubscriptionReceivedMessages.length).to./* (still) */ equal(1); - }, - realtime, - ); + expect(filteredSubscriptionReceivedMessages.length).to./* (still) */ equal(1); + }, realtime); }); }); }); diff --git a/test/common/modules/shared_helper.js b/test/common/modules/shared_helper.js index 4ce973bb45..38c7912476 100644 --- a/test/common/modules/shared_helper.js +++ b/test/common/modules/shared_helper.js @@ -171,6 +171,14 @@ define([ return result; } + async monitorConnectionThenCloseAndFinish(action, realtime, states) { + try { + await this.monitorConnectionAsync(action, realtime, states); + } finally { + await this.closeAndFinishAsync(realtime); + } + } + monitorConnection(done, realtime, states) { (states || ['failed', 'suspended']).forEach(function (state) { realtime.connection.on(state, function () { diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index cda9b88c70..e24d6a0028 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -14,14 +14,6 @@ define(['ably', 'shared_helper', 'async', 'chai', 'live_objects'], function ( return helper.AblyRealtime({ ...options, plugins: { LiveObjects: LiveObjectsPlugin } }); } - async function monitorConnectionThenCloseAndFinish(helper, action, realtime, states) { - try { - await helper.monitorConnectionAsync(action, realtime, states); - } finally { - await helper.closeAndFinishAsync(realtime); - } - } - describe('realtime/live_objects', function () { this.timeout(60 * 1000); From e52665cb8a41bad0d5c1c0a97dffeaee302080bd Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 2 Oct 2024 04:01:12 +0100 Subject: [PATCH 003/166] Add abstract LiveObject class with base shared functionality for Live Objects Resolves DTP-952 --- scripts/moduleReport.ts | 2 +- src/plugins/liveobjects/liveobject.ts | 33 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 src/plugins/liveobjects/liveobject.ts diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index f5e0f93f34..3714bc93c1 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -308,7 +308,7 @@ async function checkLiveObjectsPluginFiles() { const pluginBundleInfo = getBundleInfo('./build/liveobjects.js'); // These are the files that are allowed to contribute >= `threshold` bytes to the LiveObjects bundle. - const allowedFiles = new Set(['src/plugins/liveobjects/index.ts']); + const allowedFiles = new Set(['src/plugins/liveobjects/index.ts', 'src/plugins/liveobjects/liveobject.ts']); return checkBundleFiles(pluginBundleInfo, allowedFiles, 100); } diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts new file mode 100644 index 0000000000..945e09ced2 --- /dev/null +++ b/src/plugins/liveobjects/liveobject.ts @@ -0,0 +1,33 @@ +import { LiveObjects } from './liveobjects'; + +interface LiveObjectData { + data: any; +} + +export abstract class LiveObject { + protected _dataRef: T; + protected _objectId: string; + + constructor( + protected _liveObjects: LiveObjects, + initialData?: T | null, + objectId?: string, + ) { + this._dataRef = initialData ?? this._getZeroValueData(); + this._objectId = objectId ?? this._createObjectId(); + } + + /** + * @internal + */ + getObjectId(): string { + return this._objectId; + } + + private _createObjectId(): string { + // TODO: implement object id generation based on live object type and initial value + return Math.random().toString().substring(2); + } + + protected abstract _getZeroValueData(): T; +} From 96049bec2f66d90f0ed66ae8e58cc7f491ff00ef Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 2 Oct 2024 04:03:03 +0100 Subject: [PATCH 004/166] Add LiveMap and LiveCounter concrete classes Decoupling between underlying data and client-held reference will be achieved using `_dataRef` property on ancestor LiveObject class. Resolves DTP-953 --- src/plugins/liveobjects/livecounter.ts | 11 +++++++++ src/plugins/liveobjects/livemap.ts | 34 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 src/plugins/liveobjects/livecounter.ts create mode 100644 src/plugins/liveobjects/livemap.ts diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts new file mode 100644 index 0000000000..276fd6b991 --- /dev/null +++ b/src/plugins/liveobjects/livecounter.ts @@ -0,0 +1,11 @@ +import { LiveObject } from './liveobject'; + +export interface LiveCounterData { + data: number; +} + +export class LiveCounter extends LiveObject { + protected _getZeroValueData(): LiveCounterData { + return { data: 0 }; + } +} diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts new file mode 100644 index 0000000000..331d4403e4 --- /dev/null +++ b/src/plugins/liveobjects/livemap.ts @@ -0,0 +1,34 @@ +import { LiveObject } from './liveobject'; + +export type StateValue = string | number | boolean | Uint8Array; + +export interface ObjectIdStateData { + /** + * A reference to another state object, used to support composable state objects. + */ + objectId: string; +} + +export interface ValueStateData { + /** + * A concrete leaf value in the state object graph. + */ + value: StateValue; +} + +export type StateData = ObjectIdStateData | ValueStateData; + +export interface MapEntry { + // TODO: add tombstone, timeserial + data: StateData; +} + +export interface LiveMapData { + data: Map; +} + +export class LiveMap extends LiveObject { + protected _getZeroValueData(): LiveMapData { + return { data: new Map() }; + } +} From a63a41529b6ae6cce97387665fa84f2b1fd86ad0 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 2 Oct 2024 04:07:09 +0100 Subject: [PATCH 005/166] Add LiveObjectsPool class to store pool of live objects --- scripts/moduleReport.ts | 6 +++++- src/plugins/liveobjects/liveobjectspool.ts | 25 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 src/plugins/liveobjects/liveobjectspool.ts diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index 3714bc93c1..7183619c60 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -308,7 +308,11 @@ async function checkLiveObjectsPluginFiles() { const pluginBundleInfo = getBundleInfo('./build/liveobjects.js'); // These are the files that are allowed to contribute >= `threshold` bytes to the LiveObjects bundle. - const allowedFiles = new Set(['src/plugins/liveobjects/index.ts', 'src/plugins/liveobjects/liveobject.ts']); + const allowedFiles = new Set([ + 'src/plugins/liveobjects/index.ts', + 'src/plugins/liveobjects/liveobject.ts', + 'src/plugins/liveobjects/liveobjectspool.ts', + ]); return checkBundleFiles(pluginBundleInfo, allowedFiles, 100); } diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts new file mode 100644 index 0000000000..3431992c1f --- /dev/null +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -0,0 +1,25 @@ +import { LiveMap } from './livemap'; +import { LiveObject } from './liveobject'; +import { LiveObjects } from './liveobjects'; + +export type ObjectId = string; +export const ROOT_OBJECT_ID = 'root'; + +export class LiveObjectsPool { + private _pool: Map; + + constructor(private _liveObjects: LiveObjects) { + this._pool = this._getInitialPool(); + } + + get(objectId: ObjectId): LiveObject | undefined { + return this._pool.get(objectId); + } + + private _getInitialPool(): Map { + const pool = new Map(); + const root = new LiveMap(this._liveObjects, null, ROOT_OBJECT_ID); + pool.set(root.getObjectId(), root); + return pool; + } +} From d7211a48694a890f9448c7477ed6aec27d7afd75 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 2 Oct 2024 04:07:50 +0100 Subject: [PATCH 006/166] Add LiveObjectsPool to LiveObjects and naive implementation of `getRoot` method --- scripts/moduleReport.ts | 1 + src/plugins/liveobjects/liveobjects.ts | 9 ++++++ test/common/modules/private_api_recorder.js | 1 + test/realtime/live_objects.test.js | 31 +++++++++++++++++++++ 4 files changed, 42 insertions(+) diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index 7183619c60..dce162b9ca 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -311,6 +311,7 @@ async function checkLiveObjectsPluginFiles() { const allowedFiles = new Set([ 'src/plugins/liveobjects/index.ts', 'src/plugins/liveobjects/liveobject.ts', + 'src/plugins/liveobjects/liveobjects.ts', 'src/plugins/liveobjects/liveobjectspool.ts', ]); diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 6dfed511f5..6ba94384fd 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -1,12 +1,21 @@ import type BaseClient from 'common/lib/client/baseclient'; import type RealtimeChannel from 'common/lib/client/realtimechannel'; +import { LiveMap } from './livemap'; +import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; export class LiveObjects { private _client: BaseClient; private _channel: RealtimeChannel; + private _liveObjectsPool: LiveObjectsPool; constructor(channel: RealtimeChannel) { this._channel = channel; this._client = channel.client; + this._liveObjectsPool = new LiveObjectsPool(this); + } + + async getRoot(): Promise { + // TODO: wait for SYNC sequence to finish to return root + return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; } } diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index 57cc6c55d3..848004242e 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -44,6 +44,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'call.http._getHosts', 'call.http.checkConnectivity', 'call.http.doUri', + 'call.LiveObject.getObjectId', 'call.msgpack.decode', 'call.msgpack.encode', 'call.presence._myMembers.put', diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index e24d6a0028..e66a882a87 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -47,6 +47,37 @@ define(['ably', 'shared_helper', 'async', 'chai', 'live_objects'], function ( const channel = client.channels.get('channel'); expect(channel.liveObjects.constructor.name).to.equal('LiveObjects'); }); + + describe('LiveObjects instance', () => { + /** @nospec */ + it('getRoot() returns LiveMap instance', async function () { + const helper = this.test.helper; + const client = LiveObjectsRealtime(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get('channel'); + const liveObjects = channel.liveObjects; + const root = await liveObjects.getRoot(); + + expect(root.constructor.name).to.equal('LiveMap'); + }, client); + }); + + /** @nospec */ + it('getRoot() returns live object with id "root"', async function () { + const helper = this.test.helper; + const client = LiveObjectsRealtime(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get('channel'); + const liveObjects = channel.liveObjects; + const root = await liveObjects.getRoot(); + + helper.recordPrivateApi('call.LiveObject.getObjectId'); + expect(root.getObjectId()).to.equal('root'); + }, client); + }); + }); }); }); }); From ae331d9209852081b4cc1ccf08e7dd4d62c50ae4 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 2 Oct 2024 04:13:46 +0100 Subject: [PATCH 007/166] Implement LiveCounter access API Resolves DTP-960 --- src/plugins/liveobjects/livecounter.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 276fd6b991..fbc9ac7d98 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -5,6 +5,10 @@ export interface LiveCounterData { } export class LiveCounter extends LiveObject { + value(): number { + return this._dataRef.data; + } + protected _getZeroValueData(): LiveCounterData { return { data: 0 }; } From e5f0d8fe7fb599ba6b16b92309dbdb2b5f88c69e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 2 Oct 2024 04:28:07 +0100 Subject: [PATCH 008/166] Implement LiveMap access API Resolves DTP-961 --- scripts/moduleReport.ts | 1 + src/plugins/liveobjects/livemap.ts | 21 +++++++++++++++++++++ src/plugins/liveobjects/liveobjects.ts | 7 +++++++ 3 files changed, 29 insertions(+) diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index dce162b9ca..671f75ce8a 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -310,6 +310,7 @@ async function checkLiveObjectsPluginFiles() { // These are the files that are allowed to contribute >= `threshold` bytes to the LiveObjects bundle. const allowedFiles = new Set([ 'src/plugins/liveobjects/index.ts', + 'src/plugins/liveobjects/livemap.ts', 'src/plugins/liveobjects/liveobject.ts', 'src/plugins/liveobjects/liveobjects.ts', 'src/plugins/liveobjects/liveobjectspool.ts', diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 331d4403e4..8e2696219a 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -28,6 +28,27 @@ export interface LiveMapData { } export class LiveMap extends LiveObject { + /** + * Returns the value associated with the specified key in the underlying Map object. + * If no element is associated with the specified key, undefined is returned. + * If the value that is associated to the provided key is an objectId string of another Live Object, + * then you will get a reference to that Live Object if it exists in the local pool, or undefined otherwise. + * If the value is not an objectId, then you will get that value. + */ + get(key: string): LiveObject | StateValue | undefined { + const element = this._dataRef.data.get(key); + + if (element === undefined) { + return undefined; + } + + if ('value' in element.data) { + return element.data.value; + } else { + return this._liveObjects.getPool().get(element.data.objectId); + } + } + protected _getZeroValueData(): LiveMapData { return { data: new Map() }; } diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 6ba94384fd..9ade0b6b2e 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -18,4 +18,11 @@ export class LiveObjects { // TODO: wait for SYNC sequence to finish to return root return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; } + + /** + * @internal + */ + getPool(): LiveObjectsPool { + return this._liveObjectsPool; + } } From 8efba440f5c36a1ffc13d671638dd9dc73641827 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Oct 2024 02:43:35 +0100 Subject: [PATCH 009/166] Add STATE_SUBSCRIBE and STATE_PUBLISH channel modes --- ably.d.ts | 10 ++++++++++ src/common/lib/types/protocolmessage.ts | 19 +++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/ably.d.ts b/ably.d.ts index 18b7ee7208..2f3dd499af 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -871,6 +871,14 @@ declare namespace ChannelModes { * The client can receive presence messages. */ type PRESENCE_SUBSCRIBE = 'PRESENCE_SUBSCRIBE'; + /** + * The client can publish LiveObjects state messages. + */ + type STATE_PUBLISH = 'STATE_PUBLISH'; + /** + * The client can receive LiveObjects state messages. + */ + type STATE_SUBSCRIBE = 'STATE_SUBSCRIBE'; /** * The client is resuming an existing connection. */ @@ -885,6 +893,8 @@ export type ChannelMode = | ChannelModes.SUBSCRIBE | ChannelModes.PRESENCE | ChannelModes.PRESENCE_SUBSCRIBE + | ChannelModes.STATE_PUBLISH + | ChannelModes.STATE_SUBSCRIBE | ChannelModes.ATTACH_RESUME; /** diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index eaa622a8d9..ef155129c9 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -48,9 +48,17 @@ const flags: { [key: string]: number } = { PUBLISH: 1 << 17, SUBSCRIBE: 1 << 18, PRESENCE_SUBSCRIBE: 1 << 19, + STATE_SUBSCRIBE: 1 << 24, + STATE_PUBLISH: 1 << 25, }; const flagNames = Object.keys(flags); -flags.MODE_ALL = flags.PRESENCE | flags.PUBLISH | flags.SUBSCRIBE | flags.PRESENCE_SUBSCRIBE; +flags.MODE_ALL = + flags.PRESENCE | + flags.PUBLISH | + flags.SUBSCRIBE | + flags.PRESENCE_SUBSCRIBE | + flags.STATE_SUBSCRIBE | + flags.STATE_PUBLISH; function toStringArray(array?: any[]): string { const result = []; @@ -62,7 +70,14 @@ function toStringArray(array?: any[]): string { return '[ ' + result.join(', ') + ' ]'; } -export const channelModes = ['PRESENCE', 'PUBLISH', 'SUBSCRIBE', 'PRESENCE_SUBSCRIBE']; +export const channelModes = [ + 'PRESENCE', + 'PUBLISH', + 'SUBSCRIBE', + 'PRESENCE_SUBSCRIBE', + 'STATE_SUBSCRIBE', + 'STATE_PUBLISH', +]; export const serialize = Utils.encodeBody; From 8146caeb58041a9aa450f57147f3f7e47db3d640 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Oct 2024 04:06:53 +0100 Subject: [PATCH 010/166] Change internal `fromDeserializedIncludingDependencies` to `makeFromDeserializedWithDependencies` This is in preparation for following changes where `fromDeserialized` function would require a LiveObjectsPlugin to create StateMessage classes. We can't include LiveObjectsPlugin in the core library, so this plugin will need to be provided by the tests. --- src/common/lib/types/protocolmessage.ts | 14 +++++++++++--- src/platform/nativescript/index.ts | 4 ++-- src/platform/nodejs/index.ts | 4 ++-- src/platform/react-native/index.ts | 4 ++-- src/platform/web/index.ts | 5 +++-- test/common/modules/private_api_recorder.js | 2 +- test/realtime/channel.test.js | 12 ++++++------ test/realtime/connection.test.js | 4 ++-- test/realtime/failure.test.js | 4 ++-- test/realtime/live_objects.test.js | 2 +- test/realtime/message.test.js | 4 ++-- test/realtime/presence.test.js | 4 ++-- test/realtime/sync.test.js | 12 ++++++------ 13 files changed, 42 insertions(+), 33 deletions(-) diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index ef155129c9..91de1169c8 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -8,6 +8,7 @@ import PresenceMessage, { fromValues as presenceMessageFromValues, fromValuesArray as presenceMessagesFromValuesArray, } from './presencemessage'; +import type * as LiveObjectsPlugin from 'plugins/liveobjects'; export const actions = { HEARTBEAT: 0, @@ -110,10 +111,17 @@ export function fromDeserialized( } /** - * Used by the tests. + * Used internally by the tests. + * + * LiveObjectsPlugin code can't be included as part of the core library to prevent size growth, + * so if a test needs to build Live Object state messages, then it must provide LiveObjectsPlugin. */ -export function fromDeserializedIncludingDependencies(deserialized: Record): ProtocolMessage { - return fromDeserialized(deserialized, { presenceMessageFromValues, presenceMessagesFromValuesArray }); +export function makeFromDeserializedWithDependencies(dependencies?: { + LiveObjectsPlugin: typeof LiveObjectsPlugin | null; +}) { + return (deserialized: Record): ProtocolMessage => { + return fromDeserialized(deserialized, { presenceMessageFromValues, presenceMessagesFromValuesArray }); + }; } export function fromValues(values: unknown): ProtocolMessage { diff --git a/src/platform/nativescript/index.ts b/src/platform/nativescript/index.ts index 5a57dbe073..448d513d78 100644 --- a/src/platform/nativescript/index.ts +++ b/src/platform/nativescript/index.ts @@ -3,7 +3,7 @@ import { DefaultRest } from '../../common/lib/client/defaultrest'; import { DefaultRealtime } from '../../common/lib/client/defaultrealtime'; import Platform from '../../common/platform'; import ErrorInfo from '../../common/lib/types/errorinfo'; -import { fromDeserializedIncludingDependencies as protocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; +import { makeFromDeserializedWithDependencies as makeProtocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; // Platform Specific import BufferUtils from '../web/lib/util/bufferutils'; @@ -52,5 +52,5 @@ export default { Rest: DefaultRest, Realtime: DefaultRealtime, msgpack, - protocolMessageFromDeserialized, + makeProtocolMessageFromDeserialized, }; diff --git a/src/platform/nodejs/index.ts b/src/platform/nodejs/index.ts index 057d412e66..cca312e1e7 100644 --- a/src/platform/nodejs/index.ts +++ b/src/platform/nodejs/index.ts @@ -3,7 +3,7 @@ import { DefaultRest } from '../../common/lib/client/defaultrest'; import { DefaultRealtime } from '../../common/lib/client/defaultrealtime'; import Platform from '../../common/platform'; import ErrorInfo from '../../common/lib/types/errorinfo'; -import { fromDeserializedIncludingDependencies as protocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; +import { makeFromDeserializedWithDependencies as makeProtocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; // Platform Specific import BufferUtils from './lib/util/bufferutils'; @@ -46,5 +46,5 @@ module.exports = { Rest: DefaultRest, Realtime: DefaultRealtime, msgpack: null, - protocolMessageFromDeserialized, + makeProtocolMessageFromDeserialized, }; diff --git a/src/platform/react-native/index.ts b/src/platform/react-native/index.ts index 4153714d3a..79914a5b67 100644 --- a/src/platform/react-native/index.ts +++ b/src/platform/react-native/index.ts @@ -3,7 +3,7 @@ import { DefaultRest } from '../../common/lib/client/defaultrest'; import { DefaultRealtime } from '../../common/lib/client/defaultrealtime'; import Platform from '../../common/platform'; import ErrorInfo from '../../common/lib/types/errorinfo'; -import { fromDeserializedIncludingDependencies as protocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; +import { makeFromDeserializedWithDependencies as makeProtocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; // Platform Specific import BufferUtils from '../web/lib/util/bufferutils'; @@ -55,5 +55,5 @@ export default { Rest: DefaultRest, Realtime: DefaultRealtime, msgpack, - protocolMessageFromDeserialized, + makeProtocolMessageFromDeserialized, }; diff --git a/src/platform/web/index.ts b/src/platform/web/index.ts index f262373c8c..f1c0ddd57b 100644 --- a/src/platform/web/index.ts +++ b/src/platform/web/index.ts @@ -3,7 +3,7 @@ import { DefaultRest } from '../../common/lib/client/defaultrest'; import { DefaultRealtime } from '../../common/lib/client/defaultrealtime'; import Platform from '../../common/platform'; import ErrorInfo from '../../common/lib/types/errorinfo'; -import { fromDeserializedIncludingDependencies as protocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; +import { makeFromDeserializedWithDependencies as makeProtocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; // Platform Specific import BufferUtils from './lib/util/bufferutils'; @@ -45,11 +45,12 @@ if (Platform.Config.agent) { Platform.Defaults.agent += ' ' + Platform.Config.agent; } -export { DefaultRest as Rest, DefaultRealtime as Realtime, msgpack, protocolMessageFromDeserialized, ErrorInfo }; +export { DefaultRest as Rest, DefaultRealtime as Realtime, msgpack, makeProtocolMessageFromDeserialized, ErrorInfo }; export default { ErrorInfo, Rest: DefaultRest, Realtime: DefaultRealtime, msgpack, + makeProtocolMessageFromDeserialized, }; diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index 848004242e..af478b713f 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -49,7 +49,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'call.msgpack.encode', 'call.presence._myMembers.put', 'call.presence.waitSync', - 'call.protocolMessageFromDeserialized', + 'call.makeProtocolMessageFromDeserialized', 'call.realtime.baseUri', 'call.rest.baseUri', 'call.rest.http.do', diff --git a/test/realtime/channel.test.js b/test/realtime/channel.test.js index 7d2f5a76f2..92c1777a3e 100644 --- a/test/realtime/channel.test.js +++ b/test/realtime/channel.test.js @@ -4,7 +4,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async var exports = {}; var _exports = {}; var expect = chai.expect; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); function checkCanSubscribe(channel, testChannel) { return function (callback) { @@ -1259,7 +1259,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); var transport = realtime.connection.connectionManager.activeProtocol.getTransport(); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); transport.onProtocolMessage( createPM({ @@ -1309,7 +1309,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async } helper.recordPrivateApi('call.Platform.nextTick'); Ably.Realtime.Platform.Config.nextTick(function () { - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); transport.onProtocolMessage( createPM({ @@ -1360,7 +1360,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); var transport = realtime.connection.connectionManager.activeProtocol.getTransport(); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); transport.onProtocolMessage( createPM({ @@ -1408,7 +1408,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); var transport = realtime.connection.connectionManager.activeProtocol.getTransport(); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); transport.onProtocolMessage( createPM({ @@ -1614,7 +1614,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async setTimeout(function () { helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); var transport = realtime.connection.connectionManager.activeProtocol.getTransport(); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); transport.onProtocolMessage(createPM({ action: 11, channel: channelName })); }, 0); diff --git a/test/realtime/connection.test.js b/test/realtime/connection.test.js index c8551286c6..82c506ae50 100644 --- a/test/realtime/connection.test.js +++ b/test/realtime/connection.test.js @@ -2,7 +2,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async, chai) { var expect = chai.expect; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); describe('realtime/connection', function () { this.timeout(60 * 1000); @@ -336,7 +336,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); helper.recordPrivateApi('call.transport.onProtocolMessage'); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); connectionManager.activeProtocol.getTransport().onProtocolMessage( createPM({ action: 4, diff --git a/test/realtime/failure.test.js b/test/realtime/failure.test.js index 970e7b90e4..8b59c51e73 100644 --- a/test/realtime/failure.test.js +++ b/test/realtime/failure.test.js @@ -3,7 +3,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async, chai) { var expect = chai.expect; var noop = function () {}; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); describe('realtime/failure', function () { this.timeout(60 * 1000); @@ -644,7 +644,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async helper.closeAndFinish(done, realtime); }); }); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); helper.recordPrivateApi('call.transport.onProtocolMessage'); connectionManager.activeProtocol.getTransport().onProtocolMessage( diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index e66a882a87..ad6ae38958 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -8,7 +8,7 @@ define(['ably', 'shared_helper', 'async', 'chai', 'live_objects'], function ( LiveObjectsPlugin, ) { var expect = chai.expect; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized({ LiveObjectsPlugin }); function LiveObjectsRealtime(helper, options) { return helper.AblyRealtime({ ...options, plugins: { LiveObjects: LiveObjectsPlugin } }); diff --git a/test/realtime/message.test.js b/test/realtime/message.test.js index 6d0fe8f7c5..f56946a202 100644 --- a/test/realtime/message.test.js +++ b/test/realtime/message.test.js @@ -3,7 +3,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async, chai) { var expect = chai.expect; let config = Ably.Realtime.Platform.Config; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); var publishIntervalHelper = function (currentMessageNum, channel, dataFn, onPublish) { return function () { @@ -1146,7 +1146,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async helper.recordPrivateApi('write.connectionManager.connectionDetails.maxMessageSize'); connectionDetails.maxMessageSize = 64; helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); connectionManager.activeProtocol.getTransport().onProtocolMessage( createPM({ diff --git a/test/realtime/presence.test.js b/test/realtime/presence.test.js index f2de186fdc..2be0762da9 100644 --- a/test/realtime/presence.test.js +++ b/test/realtime/presence.test.js @@ -2,7 +2,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async, chai) { var expect = chai.expect; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); var PresenceMessage = Ably.Realtime.PresenceMessage; function extractClientIds(presenceSet) { @@ -2096,7 +2096,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async cb(); }); /* Inject an ATTACHED with RESUMED and HAS_PRESENCE both false */ - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); channel.processMessage( createPM({ action: 11, diff --git a/test/realtime/sync.test.js b/test/realtime/sync.test.js index dccbdeff52..2f3a4a9af5 100644 --- a/test/realtime/sync.test.js +++ b/test/realtime/sync.test.js @@ -2,7 +2,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async, chai) { var expect = chai.expect; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); describe('realtime/sync', function () { this.timeout(60 * 1000); @@ -50,7 +50,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channelName = 'syncexistingset', channel = realtime.channels.get(channelName); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.channel.processMessage'); await channel.processMessage( createPM({ @@ -179,7 +179,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channelName = 'sync_member_arrives_in_middle', channel = realtime.channels.get(channelName); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.channel.processMessage'); await channel.processMessage( createPM({ @@ -291,7 +291,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channelName = 'sync_member_arrives_normally_after_came_in_sync', channel = realtime.channels.get(channelName); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.channel.processMessage'); await channel.processMessage( createPM({ @@ -383,7 +383,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channelName = 'sync_member_arrives_normally_before_comes_in_sync', channel = realtime.channels.get(channelName); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.channel.processMessage'); await channel.processMessage( createPM({ @@ -479,7 +479,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channelName = 'sync_ordering', channel = realtime.channels.get(channelName); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.channel.processMessage'); await channel.processMessage( createPM({ From e6e70d9f6f4c9b2273c201770b92c21ddf3e27f4 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Oct 2024 04:20:28 +0100 Subject: [PATCH 011/166] Add STATE and STATE_SYNC protocol message actions --- src/common/lib/types/protocolmessage.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index 91de1169c8..350cc2d55c 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -30,6 +30,8 @@ export const actions = { SYNC: 16, AUTH: 17, ACTIVATE: 18, + STATE: 19, + STATE_SYNC: 20, }; export const ActionName: string[] = []; From 4f6fb8daf8290329bceb5bdbc461267a39268c4a Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Oct 2024 04:25:22 +0100 Subject: [PATCH 012/166] Add HAS_STATE flag --- src/common/lib/types/protocolmessage.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index 350cc2d55c..c9a949e483 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -53,6 +53,7 @@ const flags: { [key: string]: number } = { PRESENCE_SUBSCRIBE: 1 << 19, STATE_SUBSCRIBE: 1 << 24, STATE_PUBLISH: 1 << 25, + HAS_STATE: 1 << 26, }; const flagNames = Object.keys(flags); flags.MODE_ALL = From 9a691c4b12c4d71734617357d3eeea9dda5e6664 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Oct 2024 06:44:32 +0100 Subject: [PATCH 013/166] Add StateMessage to LiveObjects plugin --- scripts/moduleReport.ts | 1 + src/plugins/liveobjects/index.ts | 4 +- src/plugins/liveobjects/statemessage.ts | 143 ++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 src/plugins/liveobjects/statemessage.ts diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index 671f75ce8a..ff2053b60d 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -314,6 +314,7 @@ async function checkLiveObjectsPluginFiles() { 'src/plugins/liveobjects/liveobject.ts', 'src/plugins/liveobjects/liveobjects.ts', 'src/plugins/liveobjects/liveobjectspool.ts', + 'src/plugins/liveobjects/statemessage.ts', ]); return checkBundleFiles(pluginBundleInfo, allowedFiles, 100); diff --git a/src/plugins/liveobjects/index.ts b/src/plugins/liveobjects/index.ts index ff1d234a77..350024ae94 100644 --- a/src/plugins/liveobjects/index.ts +++ b/src/plugins/liveobjects/index.ts @@ -1,7 +1,9 @@ import { LiveObjects } from './liveobjects'; +import { StateMessage } from './statemessage'; -export { LiveObjects }; +export { LiveObjects, StateMessage }; export default { LiveObjects, + StateMessage, }; diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts new file mode 100644 index 0000000000..79ac63cfa1 --- /dev/null +++ b/src/plugins/liveobjects/statemessage.ts @@ -0,0 +1,143 @@ +export enum StateOperationAction { + MAP_CREATE = 0, + MAP_SET = 1, + MAP_REMOVE = 2, + COUNTER_CREATE = 3, + COUNTER_INC = 4, +} + +export enum MapSemantics { + LWW = 0, +} + +/** A StateValue represents a concrete leaf value in a state object graph. */ +export type StateValue = string | number | boolean | Buffer | Uint8Array; + +/** StateData captures a value in a state object. */ +export interface StateData { + /** A reference to another state object, used to support composable state objects. */ + objectId?: string; + /** + * The encoding the client should use to interpret the value. + * Analogous to the `encoding` field on the `Message` and `PresenceMessage` types. + */ + encoding?: string; + /** A concrete leaf value in the state object graph. */ + value?: StateValue; +} + +/** A StateMapOp describes an operation to be applied to a Map object. */ +export interface StateMapOp { + /** The key of the map entry to which the operation should be applied. */ + key: string; + /** The data that the map entry should contain if the operation is a MAP_SET operation. */ + data?: StateData; +} + +/** A StateCounterOp describes an operation to be applied to a Counter object. */ +export interface StateCounterOp { + /** The data value that should be added to the counter */ + amount: number; +} + +/** A MapEntry represents the value at a given key in a Map object. */ +export interface StateMapEntry { + /** Indicates whether the map entry has been removed. */ + tombstone?: boolean; + /** The *origin* timeserial of the last operation that was applied to the map entry. */ + timeserial: string; + /** The data that represents the value of the map entry. */ + data: StateData; +} + +/** A Map object represents a map of key-value pairs. */ +export interface StateMap { + /** The conflict-resolution semantics used by the map object. */ + semantics?: MapSemantics; + // The map entries, indexed by key. + entries?: Record; +} + +/** A Counter object represents an incrementable and decrementable value */ +export interface StateCounter { + /** The value of the counter */ + count?: number; + /** + * Indicates (true) if the counter has seen an explicit create operation + * and false if the counter was created with a default value when + * processing a regular operation. + */ + created: boolean; +} + +/** A StateOperation describes an operation to be applied to a state object. */ +export interface StateOperation { + /** Defines the operation to be applied to the state object. */ + action: StateOperationAction; + /** The object ID of the state object to which the operation should be applied. */ + objectId: string; + /** The payload for the operation if it is an operation on a Map object type. */ + mapOp?: StateMapOp; + /** The payload for the operation if it is an operation on a Counter object type. */ + counterOp?: StateCounterOp; + /** + * The payload for the operation if the operation is MAP_CREATE. + * Defines the initial value for the map object. + */ + map?: StateMap; + /** + * The payload for the operation if the operation is COUNTER_CREATE. + * Defines the initial value for the counter object. + */ + counter?: StateCounter; + /** + * The nonce, must be present on create operations. This is the random part + * that has been hashed with the type and initial value to create the object ID. + */ + nonce?: string; +} + +/** A StateObject describes the instantaneous state of an object. */ +export interface StateObject { + /** The identifier of the state object. */ + objectId: string; + /** The *regional* timeserial of the last operation that was applied to this state object. */ + regionalTimeserial: string; + /** The data that represents the state of the object if it is a Map object type. */ + map?: StateMap; + /** The data that represents the state of the object if it is a Counter object type. */ + counter?: StateCounter; +} + +/** + * @internal + */ +export class StateMessage { + id?: string; + timestamp?: number; + clientId?: string; + connectionId?: string; + channel?: string; + extras?: any; + /** Describes an operation to be applied to a state object. */ + operation?: StateOperation; + /** Describes the instantaneous state of an object. */ + object?: StateObject; + /** Timeserial format */ + serial?: string; + + static fromValues(values: StateMessage | Record): StateMessage { + return Object.assign(new StateMessage(), values); + } + + static fromValuesArray(values: unknown[]): StateMessage[] { + const count = values.length; + const result = new Array(count); + + for (let i = 0; i < count; i++) { + result[i] = this.fromValues(values[i] as Record); + } + + return result; + } +} From 8e1339d5e6ba7b1c40d912ddf232265ed12850a0 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Oct 2024 06:43:09 +0100 Subject: [PATCH 014/166] Add support for state messages to ProtocolMessage --- src/common/lib/client/baserealtime.ts | 3 ++ src/common/lib/transport/comettransport.ts | 6 ++- src/common/lib/transport/connectionmanager.ts | 3 +- src/common/lib/transport/protocol.ts | 6 ++- src/common/lib/transport/transport.ts | 6 ++- .../lib/transport/websockettransport.ts | 1 + src/common/lib/types/protocolmessage.ts | 40 ++++++++++++++++--- 7 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/common/lib/client/baserealtime.ts b/src/common/lib/client/baserealtime.ts index 8afbd429df..d6b7c35e94 100644 --- a/src/common/lib/client/baserealtime.ts +++ b/src/common/lib/client/baserealtime.ts @@ -13,12 +13,14 @@ import { ModularPlugins, RealtimePresencePlugin } from './modularplugins'; import { TransportNames } from 'common/constants/TransportName'; import { TransportImplementations } from 'common/platform'; import Defaults from '../util/defaults'; +import type * as LiveObjectsPlugin from 'plugins/liveobjects'; /** `BaseRealtime` is an export of the tree-shakable version of the SDK, and acts as the base class for the `DefaultRealtime` class exported by the non tree-shakable version. */ class BaseRealtime extends BaseClient { readonly _RealtimePresence: RealtimePresencePlugin | null; + readonly _LiveObjectsPlugin: typeof LiveObjectsPlugin | null; // Extra transport implementations available to this client, in addition to those in Platform.Transports.bundledImplementations readonly _additionalTransportImplementations: TransportImplementations; _channels: any; @@ -58,6 +60,7 @@ class BaseRealtime extends BaseClient { this._additionalTransportImplementations = BaseRealtime.transportImplementationsFromPlugins(this.options.plugins); this._RealtimePresence = this.options.plugins?.RealtimePresence ?? null; + this._LiveObjectsPlugin = this.options.plugins?.LiveObjects ?? null; this.connection = new Connection(this, this.options); this._channels = new Channels(this); if (this.options.autoConnect !== false) this.connect(); diff --git a/src/common/lib/transport/comettransport.ts b/src/common/lib/transport/comettransport.ts index ab468b6ccd..00ecb804d8 100644 --- a/src/common/lib/transport/comettransport.ts +++ b/src/common/lib/transport/comettransport.ts @@ -353,7 +353,11 @@ abstract class CometTransport extends Transport { if (items && items.length) for (let i = 0; i < items.length; i++) this.onProtocolMessage( - protocolMessageFromDeserialized(items[i], this.connectionManager.realtime._RealtimePresence), + protocolMessageFromDeserialized( + items[i], + this.connectionManager.realtime._RealtimePresence, + this.connectionManager.realtime._LiveObjectsPlugin, + ), ); } catch (e) { Logger.logAction( diff --git a/src/common/lib/transport/connectionmanager.ts b/src/common/lib/transport/connectionmanager.ts index 8ef56dcc2e..56364d6c95 100644 --- a/src/common/lib/transport/connectionmanager.ts +++ b/src/common/lib/transport/connectionmanager.ts @@ -1805,7 +1805,8 @@ class ConnectionManager extends EventEmitter { Logger.LOG_MICRO, 'ConnectionManager.send()', - 'queueing msg; ' + stringifyProtocolMessage(msg, this.realtime._RealtimePresence), + 'queueing msg; ' + + stringifyProtocolMessage(msg, this.realtime._RealtimePresence, this.realtime._LiveObjectsPlugin), ); } this.queue(msg, callback); diff --git a/src/common/lib/transport/protocol.ts b/src/common/lib/transport/protocol.ts index cb91a1db26..88a5947e4a 100644 --- a/src/common/lib/transport/protocol.ts +++ b/src/common/lib/transport/protocol.ts @@ -77,7 +77,11 @@ class Protocol extends EventEmitter { Logger.LOG_MICRO, 'Protocol.send()', 'sending msg; ' + - stringifyProtocolMessage(pendingMessage.message, this.transport.connectionManager.realtime._RealtimePresence), + stringifyProtocolMessage( + pendingMessage.message, + this.transport.connectionManager.realtime._RealtimePresence, + this.transport.connectionManager.realtime._LiveObjectsPlugin, + ), ); } pendingMessage.sendAttempted = true; diff --git a/src/common/lib/transport/transport.ts b/src/common/lib/transport/transport.ts index 23ad1ec059..3f298e24f5 100644 --- a/src/common/lib/transport/transport.ts +++ b/src/common/lib/transport/transport.ts @@ -128,7 +128,11 @@ abstract class Transport extends EventEmitter { 'received on ' + this.shortName + ': ' + - stringifyProtocolMessage(message, this.connectionManager.realtime._RealtimePresence) + + stringifyProtocolMessage( + message, + this.connectionManager.realtime._RealtimePresence, + this.connectionManager.realtime._LiveObjectsPlugin, + ) + '; connectionId = ' + this.connectionManager.connectionId, ); diff --git a/src/common/lib/transport/websockettransport.ts b/src/common/lib/transport/websockettransport.ts index 3e12f1d67c..a85a6f077e 100644 --- a/src/common/lib/transport/websockettransport.ts +++ b/src/common/lib/transport/websockettransport.ts @@ -140,6 +140,7 @@ class WebSocketTransport extends Transport { data, this.connectionManager.realtime._MsgPack, this.connectionManager.realtime._RealtimePresence, + this.connectionManager.realtime._LiveObjectsPlugin, this.format, ), ); diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index c9a949e483..dcd2191139 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -89,15 +89,17 @@ export function deserialize( serialized: unknown, MsgPack: MsgPack | null, presenceMessagePlugin: PresenceMessagePlugin | null, + liveObjectsPlugin: typeof LiveObjectsPlugin | null, format?: Utils.Format, ): ProtocolMessage { const deserialized = Utils.decodeBody>(serialized, MsgPack, format); - return fromDeserialized(deserialized, presenceMessagePlugin); + return fromDeserialized(deserialized, presenceMessagePlugin, liveObjectsPlugin); } export function fromDeserialized( deserialized: Record, presenceMessagePlugin: PresenceMessagePlugin | null, + liveObjectsPlugin: typeof LiveObjectsPlugin | null, ): ProtocolMessage { const error = deserialized.error; if (error) deserialized.error = ErrorInfo.fromValues(error as ErrorInfo); @@ -110,7 +112,18 @@ export function fromDeserialized( for (let i = 0; i < presence.length; i++) presence[i] = presenceMessagePlugin.presenceMessageFromValues(presence[i], true); } - return Object.assign(new ProtocolMessage(), { ...deserialized, presence }); + + let state: LiveObjectsPlugin.StateMessage[] | undefined = undefined; + if (liveObjectsPlugin) { + state = deserialized.state as LiveObjectsPlugin.StateMessage[]; + if (state) { + for (let i = 0; i < state.length; i++) { + state[i] = liveObjectsPlugin.StateMessage.fromValues(state[i]); + } + } + } + + return Object.assign(new ProtocolMessage(), { ...deserialized, presence, state }); } /** @@ -123,7 +136,11 @@ export function makeFromDeserializedWithDependencies(dependencies?: { LiveObjectsPlugin: typeof LiveObjectsPlugin | null; }) { return (deserialized: Record): ProtocolMessage => { - return fromDeserialized(deserialized, { presenceMessageFromValues, presenceMessagesFromValuesArray }); + return fromDeserialized( + deserialized, + { presenceMessageFromValues, presenceMessagesFromValuesArray }, + dependencies?.LiveObjectsPlugin ?? null, + ); }; } @@ -131,7 +148,11 @@ export function fromValues(values: unknown): ProtocolMessage { return Object.assign(new ProtocolMessage(), values); } -export function stringify(msg: any, presenceMessagePlugin: PresenceMessagePlugin | null): string { +export function stringify( + msg: any, + presenceMessagePlugin: PresenceMessagePlugin | null, + liveObjectsPlugin: typeof LiveObjectsPlugin | null, +): string { let result = '[ProtocolMessage'; if (msg.action !== undefined) result += '; action=' + ActionName[msg.action] || msg.action; @@ -145,6 +166,9 @@ export function stringify(msg: any, presenceMessagePlugin: PresenceMessagePlugin if (msg.messages) result += '; messages=' + toStringArray(messagesFromValuesArray(msg.messages)); if (msg.presence && presenceMessagePlugin) result += '; presence=' + toStringArray(presenceMessagePlugin.presenceMessagesFromValuesArray(msg.presence)); + if (msg.state && liveObjectsPlugin) { + result += '; state=' + toStringArray(liveObjectsPlugin.StateMessage.fromValuesArray(msg.state)); + } if (msg.error) result += '; error=' + ErrorInfo.fromValues(msg.error).toString(); if (msg.auth && msg.auth.accessToken) result += '; token=' + msg.auth.accessToken; if (msg.flags) result += '; flags=' + flagNames.filter(msg.hasFlag).join(','); @@ -176,8 +200,14 @@ class ProtocolMessage { channelSerial?: string | null; msgSerial?: number; messages?: Message[]; - // This will be undefined if we skipped decoding this property due to user not requesting presence functionality — see `fromDeserialized` + /** + * This will be undefined if we skipped decoding this property due to user not requesting Presence functionality — see {@link fromDeserialized} + */ presence?: PresenceMessage[]; + /** + * This will be undefined if we skipped decoding this property due to user not requesting LiveObjects functionality — see {@link fromDeserialized} + */ + state?: LiveObjectsPlugin.StateMessage[]; auth?: unknown; connectionDetails?: Record; From 59b3aefe376d22cabb8718d9ffccbdaff6bfd3bc Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Oct 2024 06:51:31 +0100 Subject: [PATCH 015/166] Implement `toJSON` method for StateMessage --- src/common/lib/types/protocolmessage.ts | 5 +- src/plugins/liveobjects/statemessage.ts | 108 +++++++++++++++++++++++- 2 files changed, 107 insertions(+), 6 deletions(-) diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index dcd2191139..74a1e64db7 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -9,6 +9,7 @@ import PresenceMessage, { fromValuesArray as presenceMessagesFromValuesArray, } from './presencemessage'; import type * as LiveObjectsPlugin from 'plugins/liveobjects'; +import Platform from '../../platform'; export const actions = { HEARTBEAT: 0, @@ -118,7 +119,7 @@ export function fromDeserialized( state = deserialized.state as LiveObjectsPlugin.StateMessage[]; if (state) { for (let i = 0; i < state.length; i++) { - state[i] = liveObjectsPlugin.StateMessage.fromValues(state[i]); + state[i] = liveObjectsPlugin.StateMessage.fromValues(state[i], Platform); } } } @@ -167,7 +168,7 @@ export function stringify( if (msg.presence && presenceMessagePlugin) result += '; presence=' + toStringArray(presenceMessagePlugin.presenceMessagesFromValuesArray(msg.presence)); if (msg.state && liveObjectsPlugin) { - result += '; state=' + toStringArray(liveObjectsPlugin.StateMessage.fromValuesArray(msg.state)); + result += '; state=' + toStringArray(liveObjectsPlugin.StateMessage.fromValuesArray(msg.state, Platform)); } if (msg.error) result += '; error=' + ErrorInfo.fromValues(msg.error).toString(); if (msg.auth && msg.auth.accessToken) result += '; token=' + msg.auth.accessToken; diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 79ac63cfa1..4cab427fa4 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -1,3 +1,5 @@ +import type Platform from 'common/platform'; + export enum StateOperationAction { MAP_CREATE = 0, MAP_SET = 1, @@ -126,18 +128,116 @@ export class StateMessage { /** Timeserial format */ serial?: string; - static fromValues(values: StateMessage | Record): StateMessage { - return Object.assign(new StateMessage(), values); + constructor(private _platform: typeof Platform) {} + + static fromValues(values: StateMessage | Record, platform: typeof Platform): StateMessage { + return Object.assign(new StateMessage(platform), values); } - static fromValuesArray(values: unknown[]): StateMessage[] { + static fromValuesArray(values: unknown[], platform: typeof Platform): StateMessage[] { const count = values.length; const result = new Array(count); for (let i = 0; i < count; i++) { - result[i] = this.fromValues(values[i] as Record); + result[i] = this.fromValues(values[i] as Record, platform); } return result; } + + /** + * Overload toJSON() to intercept JSON.stringify() + * @return {*} + */ + toJSON(): { + id?: string; + clientId?: string; + operation?: StateOperation; + object?: StateObject; + extras?: any; + } { + // need to encode buffer data to base64 if present and if we're returning a real JSON. + // although msgpack also calls toJSON() directly, + // we know it is a JSON.stringify() call if we have a non-empty arguments list. + // if withBase64Encoding = true - JSON.stringify() call + // if withBase64Encoding = false - we were called by msgpack + const withBase64Encoding = arguments.length > 0; + + let operationCopy: StateOperation | undefined = undefined; + if (this.operation) { + // deep copy "operation" prop so we can modify it here. + // buffer values won't be correctly copied, so we will need to set them again explictly + operationCopy = JSON.parse(JSON.stringify(this.operation)) as StateOperation; + + if (operationCopy.mapOp?.data && 'value' in operationCopy.mapOp.data) { + // use original "operation" prop when encoding values, so we have access to original buffer values. + operationCopy.mapOp.data = this._encodeStateData(this.operation.mapOp?.data!, withBase64Encoding); + } + + if (operationCopy.map?.entries) { + Object.entries(operationCopy.map.entries).forEach(([key, entry]) => { + // use original "operation" prop when encoding values, so we have access to original buffer values. + entry.data = this._encodeStateData(this.operation?.map?.entries?.[key].data!, withBase64Encoding); + }); + } + } + + let object: StateObject | undefined = undefined; + if (this.object) { + // deep copy "object" prop so we can modify it here. + // buffer values won't be correctly copied, so we will need to set them again explictly + object = JSON.parse(JSON.stringify(this.object)) as StateObject; + + if (object.map?.entries) { + Object.entries(object.map.entries).forEach(([key, entry]) => { + // use original "object" prop when encoding values, so we have access to original buffer values. + entry.data = this._encodeStateData(this.object?.map?.entries?.[key].data!, withBase64Encoding); + }); + } + } + + return { + id: this.id, + clientId: this.clientId, + operation: operationCopy, + object: object, + extras: this.extras, + }; + } + + private _encodeStateData(data: StateData, withBase64Encoding: boolean): StateData { + const { value, encoding } = this._encodeStateValue(data?.value, data?.encoding, withBase64Encoding); + return { + ...data, + value, + encoding, + }; + } + + private _encodeStateValue( + value: StateValue | undefined, + encoding: string | undefined, + withBase64Encoding: boolean, + ): { + value: StateValue | undefined; + encoding: string | undefined; + } { + if (!value || !this._platform.BufferUtils.isBuffer(value)) { + return { value, encoding }; + } + + if (withBase64Encoding) { + return { + value: this._platform.BufferUtils.base64Encode(value), + encoding: encoding ? encoding + '/base64' : 'base64', + }; + } + + // toBuffer returns a datatype understandable by + // that platform's msgpack implementation (Buffer in node, Uint8Array in browsers) + return { + value: this._platform.BufferUtils.toBuffer(value), + encoding, + }; + } } From 2dcf29822bd39bbb1ed6433888203198964feeda Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Oct 2024 06:56:19 +0100 Subject: [PATCH 016/166] Implement `toString` method for StateMessage --- src/plugins/liveobjects/statemessage.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 4cab427fa4..2dd257d9b7 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -205,6 +205,25 @@ export class StateMessage { }; } + toString(): string { + let result = '[StateMessage'; + + if (this.id) result += '; id=' + this.id; + if (this.timestamp) result += '; timestamp=' + this.timestamp; + if (this.clientId) result += '; clientId=' + this.clientId; + if (this.connectionId) result += '; connectionId=' + this.connectionId; + // TODO: prettify output for operation and object and encode buffers. + // see examples for data in Message and PresenceMessage + if (this.operation) result += '; operation=' + JSON.stringify(this.operation); + if (this.object) result += '; object=' + JSON.stringify(this.object); + if (this.extras) result += '; extras=' + JSON.stringify(this.extras); + if (this.serial) result += '; serial=' + this.serial; + + result += ']'; + + return result; + } + private _encodeStateData(data: StateData, withBase64Encoding: boolean): StateData { const { value, encoding } = this._encodeStateValue(data?.value, data?.encoding, withBase64Encoding); return { From 4b5177361624c461a20a45c832a05b215ed9aa81 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Oct 2024 08:02:35 +0100 Subject: [PATCH 017/166] Split Message `decode` function into `decode` and `decodeData` This is in preparation for adding decoding messages to StateMessage. StateMessage has multiple data/encoding entries and requires a more generic function to decode its data, instead of a previous `decode` function that expected data and encoding to be present on a Message class --- src/common/lib/types/message.ts | 69 ++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/src/common/lib/types/message.ts b/src/common/lib/types/message.ts index 7cc8b80aca..c2a1f1cdb1 100644 --- a/src/common/lib/types/message.ts +++ b/src/common/lib/types/message.ts @@ -154,17 +154,36 @@ export async function decode( message: Message | PresenceMessage, inputContext: CipherOptions | EncodingDecodingContext | ChannelOptions, ): Promise { + const { data, encoding, error } = await decodeData(message.data, message.encoding, inputContext); + message.data = data; + message.encoding = encoding; + + if (error) { + throw error; + } +} + +export async function decodeData( + data: any, + encoding: string | null | undefined, + inputContext: CipherOptions | EncodingDecodingContext | ChannelOptions, +): Promise<{ + error?: ErrorInfo; + data: any; + encoding: string | null | undefined; +}> { const context = normaliseContext(inputContext); + let lastPayload = data; + let decodedData = data; + let finalEncoding: string | null | undefined = encoding; + let decodingError: ErrorInfo | undefined = undefined; - let lastPayload = message.data; - const encoding = message.encoding; if (encoding) { const xforms = encoding.split('/'); - let lastProcessedEncodingIndex, - encodingsToProcess = xforms.length, - data = message.data; - + let lastProcessedEncodingIndex; + let encodingsToProcess = xforms.length; let xform = ''; + try { while ((lastProcessedEncodingIndex = encodingsToProcess) > 0) { // eslint-disable-next-line security/detect-unsafe-regex @@ -173,16 +192,16 @@ export async function decode( xform = match[1]; switch (xform) { case 'base64': - data = Platform.BufferUtils.base64Decode(String(data)); + decodedData = Platform.BufferUtils.base64Decode(String(decodedData)); if (lastProcessedEncodingIndex == xforms.length) { - lastPayload = data; + lastPayload = decodedData; } continue; case 'utf-8': - data = Platform.BufferUtils.utf8Decode(data); + decodedData = Platform.BufferUtils.utf8Decode(decodedData); continue; case 'json': - data = JSON.parse(data); + decodedData = JSON.parse(decodedData); continue; case 'cipher': if ( @@ -196,7 +215,7 @@ export async function decode( if (xformAlgorithm != cipher.algorithm) { throw new Error('Unable to decrypt message with given cipher; incompatible cipher params'); } - data = await cipher.decrypt(data); + decodedData = await cipher.decrypt(decodedData); continue; } else { throw new Error('Unable to decrypt message; not an encrypted channel'); @@ -220,10 +239,12 @@ export async function decode( // vcdiff expects Uint8Arrays, can't copy with ArrayBuffers. const deltaBaseBuffer = Platform.BufferUtils.toBuffer(deltaBase as Buffer); - data = Platform.BufferUtils.toBuffer(data); + decodedData = Platform.BufferUtils.toBuffer(decodedData); - data = Platform.BufferUtils.arrayBufferViewToBuffer(context.plugins.vcdiff.decode(data, deltaBaseBuffer)); - lastPayload = data; + decodedData = Platform.BufferUtils.arrayBufferViewToBuffer( + context.plugins.vcdiff.decode(decodedData, deltaBaseBuffer), + ); + lastPayload = decodedData; } catch (e) { throw new ErrorInfo('Vcdiff delta decode failed with ' + e, 40018, 400); } @@ -234,18 +255,30 @@ export async function decode( } } catch (e) { const err = e as ErrorInfo; - throw new ErrorInfo( - 'Error processing the ' + xform + ' encoding, decoder returned ‘' + err.message + '’', + decodingError = new ErrorInfo( + `Error processing the ${xform} encoding, decoder returned ‘${err.message}’`, err.code || 40013, 400, ); } finally { - message.encoding = + finalEncoding = (lastProcessedEncodingIndex as number) <= 0 ? null : xforms.slice(0, lastProcessedEncodingIndex).join('/'); - message.data = data; } } + + if (decodingError) { + return { + error: decodingError, + data: decodedData, + encoding: finalEncoding, + }; + } + context.baseEncodedPreviousPayload = lastPayload; + return { + data: decodedData, + encoding: finalEncoding, + }; } export async function fromResponseBody( From e8e9472fd557b773d616e09a9e4738ad744ffbf6 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Oct 2024 08:36:25 +0100 Subject: [PATCH 018/166] Implement `decode` method for StateMessage --- src/plugins/liveobjects/statemessage.ts | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 2dd257d9b7..c8063e4550 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -1,4 +1,6 @@ +import type { decodeData } from 'common/lib/types/message'; import type Platform from 'common/platform'; +import type { ChannelOptions } from 'common/types/channel'; export enum StateOperationAction { MAP_CREATE = 0, @@ -130,6 +132,51 @@ export class StateMessage { constructor(private _platform: typeof Platform) {} + static async decode( + message: StateMessage, + inputContext: ChannelOptions, + decodeDataFn: typeof decodeData, + ): Promise { + // TODO: decide how to handle individual errors from decoding values. currently we throw first ever error we get + + const decodeMapEntry = async ( + entry: StateMapEntry, + ctx: ChannelOptions, + decode: typeof decodeData, + ): Promise => { + const { data, encoding, error } = await decode(entry.data.value, entry.data.encoding, ctx); + entry.data.value = data; + entry.data.encoding = encoding ?? undefined; + + if (error) { + throw error; + } + }; + + if (message.object?.map?.entries) { + for (const entry of Object.values(message.object.map.entries)) { + decodeMapEntry(entry, inputContext, decodeDataFn); + } + } + + if (message.operation?.map) { + for (const entry of Object.values(message.operation.map)) { + decodeMapEntry(entry, inputContext, decodeDataFn); + } + } + + if (message.operation?.mapOp?.data && 'value' in message.operation?.mapOp?.data) { + const mapOpData = message.operation.mapOp.data; + const { data, encoding, error } = await decodeDataFn(mapOpData.value, mapOpData.encoding, inputContext); + mapOpData.value = data; + mapOpData.encoding = encoding ?? undefined; + + if (error) { + throw error; + } + } + } + static fromValues(values: StateMessage | Record, platform: typeof Platform): StateMessage { return Object.assign(new StateMessage(platform), values); } From 88e4736dfaf9b01f25dbc82a230cc8ff2a021497 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Oct 2024 10:45:59 +0100 Subject: [PATCH 019/166] Implement handling of the server-initiated state sync sequence STATE_SYNC message processing in `RealtimeChannel.processMessage` is based on the process for `PRESENCE` message. Resolves DTP-950 --- scripts/moduleReport.ts | 3 +- src/common/lib/client/realtimechannel.ts | 49 +++++- src/plugins/liveobjects/livecounter.ts | 4 +- src/plugins/liveobjects/livemap.ts | 7 +- src/plugins/liveobjects/liveobject.ts | 24 ++- src/plugins/liveobjects/liveobjects.ts | 149 ++++++++++++++++++ src/plugins/liveobjects/liveobjectspool.ts | 33 +++- .../liveobjects/syncliveobjectsdatapool.ts | 35 ++++ 8 files changed, 289 insertions(+), 15 deletions(-) create mode 100644 src/plugins/liveobjects/syncliveobjectsdatapool.ts diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index ff2053b60d..e260f0783d 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -6,7 +6,7 @@ import { gzip } from 'zlib'; import Table from 'cli-table'; // The maximum size we allow for a minimal useful Realtime bundle (i.e. one that can subscribe to a channel) -const minimalUsefulRealtimeBundleSizeThresholdsKiB = { raw: 99, gzip: 30 }; +const minimalUsefulRealtimeBundleSizeThresholdsKiB = { raw: 100, gzip: 31 }; const baseClientNames = ['BaseRest', 'BaseRealtime']; @@ -315,6 +315,7 @@ async function checkLiveObjectsPluginFiles() { 'src/plugins/liveobjects/liveobjects.ts', 'src/plugins/liveobjects/liveobjectspool.ts', 'src/plugins/liveobjects/statemessage.ts', + 'src/plugins/liveobjects/syncliveobjectsdatapool.ts', ]); return checkBundleFiles(pluginBundleInfo, allowedFiles, 100); diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index 1a2554f6b4..928650ad6b 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -12,6 +12,7 @@ import Message, { fromValuesArray as messagesFromValuesArray, encodeArray as encodeMessagesArray, decode as decodeMessage, + decodeData, getMessagesSize, CipherOptions, EncodingDecodingContext, @@ -533,12 +534,18 @@ class RealtimeChannel extends EventEmitter { const resumed = message.hasFlag('RESUMED'); const hasPresence = message.hasFlag('HAS_PRESENCE'); const hasBacklog = message.hasFlag('HAS_BACKLOG'); + const hasState = message.hasFlag('HAS_STATE'); if (this.state === 'attached') { if (!resumed) { - /* On a loss of continuity, the presence set needs to be re-synced */ + // we have lost continuity. + // the presence set needs to be re-synced if (this._presence) { this._presence.onAttached(hasPresence); } + // the Live Objects state needs to be re-synced + if (this._liveObjects) { + this._liveObjects.onAttached(hasState); + } } const change = new ChannelStateChange(this.state, this.state, resumed, hasBacklog, message.error); this._allChannelChanges.emit('update', change); @@ -549,7 +556,7 @@ class RealtimeChannel extends EventEmitter { /* RTL5i: re-send DETACH and remain in the 'detaching' state */ this.checkPendingState(); } else { - this.notifyState('attached', message.error, resumed, hasPresence, hasBacklog); + this.notifyState('attached', message.error, resumed, hasPresence, hasBacklog, hasState); } break; } @@ -613,6 +620,40 @@ class RealtimeChannel extends EventEmitter { } break; } + + case actions.STATE_SYNC: { + if (!this._liveObjects) { + return; + } + + const { id, connectionId, timestamp } = message; + const options = this.channelOptions; + + const stateMessages = message.state ?? []; + for (let i = 0; i < stateMessages.length; i++) { + try { + const stateMessage = stateMessages[i]; + + await this.client._LiveObjectsPlugin?.StateMessage.decode(stateMessage, options, decodeData); + + if (!stateMessage.connectionId) stateMessage.connectionId = connectionId; + if (!stateMessage.timestamp) stateMessage.timestamp = timestamp; + if (!stateMessage.id) stateMessage.id = id + ':' + i; + } catch (e) { + Logger.logAction( + this.logger, + Logger.LOG_ERROR, + 'RealtimeChannel.processMessage()', + (e as Error).toString(), + ); + } + } + + this._liveObjects.handleStateSyncMessage(stateMessages, message.channelSerial); + + break; + } + case actions.MESSAGE: { //RTL17 if (this.state !== 'attached') { @@ -743,6 +784,7 @@ class RealtimeChannel extends EventEmitter { resumed?: boolean, hasPresence?: boolean, hasBacklog?: boolean, + hasState?: boolean, ): void { Logger.logAction( this.logger, @@ -763,6 +805,9 @@ class RealtimeChannel extends EventEmitter { if (this._presence) { this._presence.actOnChannelState(state, hasPresence, reason); } + if (this._liveObjects) { + this._liveObjects.actOnChannelState(state, hasState); + } if (state === 'suspended' && this.connectionManager.state.sendEvents) { this.startRetryTimer(); } else { diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index fbc9ac7d98..06398d6e9f 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -1,6 +1,6 @@ -import { LiveObject } from './liveobject'; +import { LiveObject, LiveObjectData } from './liveobject'; -export interface LiveCounterData { +export interface LiveCounterData extends LiveObjectData { data: number; } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 8e2696219a..b147dcf80f 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -1,6 +1,5 @@ -import { LiveObject } from './liveobject'; - -export type StateValue = string | number | boolean | Uint8Array; +import { LiveObject, LiveObjectData } from './liveobject'; +import { StateValue } from './statemessage'; export interface ObjectIdStateData { /** @@ -23,7 +22,7 @@ export interface MapEntry { data: StateData; } -export interface LiveMapData { +export interface LiveMapData extends LiveObjectData { data: Map; } diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index 945e09ced2..d062fec37b 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -1,12 +1,13 @@ import { LiveObjects } from './liveobjects'; -interface LiveObjectData { +export interface LiveObjectData { data: any; } export abstract class LiveObject { protected _dataRef: T; protected _objectId: string; + protected _regionalTimeserial?: string; constructor( protected _liveObjects: LiveObjects, @@ -24,6 +25,27 @@ export abstract class LiveObject { return this._objectId; } + /** + * @internal + */ + getRegionalTimeserial(): string | undefined { + return this._regionalTimeserial; + } + + /** + * @internal + */ + setData(newDataRef: T): void { + this._dataRef = newDataRef; + } + + /** + * @internal + */ + setRegionalTimeserial(regionalTimeserial: string): void { + this._regionalTimeserial = regionalTimeserial; + } + private _createObjectId(): string { // TODO: implement object id generation based on live object type and initial value return Math.random().toString().substring(2); diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 9ade0b6b2e..bc7ce74a61 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -1,17 +1,28 @@ import type BaseClient from 'common/lib/client/baseclient'; import type RealtimeChannel from 'common/lib/client/realtimechannel'; +import type * as API from '../../../ably'; +import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; +import { LiveObject } from './liveobject'; import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; +import { StateMessage } from './statemessage'; +import { SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; export class LiveObjects { private _client: BaseClient; private _channel: RealtimeChannel; private _liveObjectsPool: LiveObjectsPool; + private _syncLiveObjectsDataPool: SyncLiveObjectsDataPool; + private _syncInProgress: boolean; + private _currentSyncId: string | undefined; + private _currentSyncCursor: string | undefined; constructor(channel: RealtimeChannel) { this._channel = channel; this._client = channel.client; this._liveObjectsPool = new LiveObjectsPool(this); + this._syncLiveObjectsDataPool = new SyncLiveObjectsDataPool(this); + this._syncInProgress = true; } async getRoot(): Promise { @@ -25,4 +36,142 @@ export class LiveObjects { getPool(): LiveObjectsPool { return this._liveObjectsPool; } + + /** + * @internal + */ + getClient(): BaseClient { + return this._client; + } + + /** + * @internal + */ + handleStateSyncMessage(stateMessages: StateMessage[], syncChannelSerial: string | null | undefined): void { + const { syncId, syncCursor } = this._parseSyncChannelSerial(syncChannelSerial); + if (this._currentSyncId !== syncId) { + this._startNewSync(syncId, syncCursor); + } + + // TODO: delegate state messages to _syncLiveObjectsDataPool and create new live and data objects + + // if this is the last (or only) message in a sequence of sync updates, end the sync + if (!syncCursor) { + this._endSync(); + } + } + + /** + * @internal + */ + onAttached(hasState?: boolean): void { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MINOR, + 'LiveObjects.onAttached()', + 'channel = ' + this._channel.name + ', hasState = ' + hasState, + ); + + if (hasState) { + this._startNewSync(undefined); + } else { + // no HAS_STATE flag received on attach, can end SYNC sequence immediately + // and treat it as no state on a channel + this._liveObjectsPool.reset(); + this._syncLiveObjectsDataPool.reset(); + this._endSync(); + } + } + + /** + * @internal + */ + actOnChannelState(state: API.ChannelState, hasState?: boolean): void { + switch (state) { + case 'attached': + this.onAttached(hasState); + break; + + case 'detached': + case 'failed': + // TODO: do something + break; + + case 'suspended': + // TODO: do something + break; + } + } + + private _startNewSync(syncId?: string, syncCursor?: string): void { + this._syncLiveObjectsDataPool.reset(); + this._currentSyncId = syncId; + this._currentSyncCursor = syncCursor; + this._syncInProgress = true; + } + + private _endSync(): void { + this._applySync(); + this._syncLiveObjectsDataPool.reset(); + this._currentSyncId = undefined; + this._currentSyncCursor = undefined; + this._syncInProgress = false; + } + + private _parseSyncChannelSerial(syncChannelSerial: string | null | undefined): { + syncId: string | undefined; + syncCursor: string | undefined; + } { + let match: RegExpMatchArray | null; + let syncId: string | undefined = undefined; + let syncCursor: string | undefined = undefined; + if (syncChannelSerial && (match = syncChannelSerial.match(/^([\w-]+):(.*)$/))) { + syncId = match[1]; + syncCursor = match[2]; + } + + return { + syncId, + syncCursor, + }; + } + + private _applySync(): void { + if (this._syncLiveObjectsDataPool.isEmpty()) { + return; + } + + const receivedObjectIds = new Set(); + + for (const [objectId, entry] of this._syncLiveObjectsDataPool.entries()) { + receivedObjectIds.add(objectId); + const existingObject = this._liveObjectsPool.get(objectId); + + if (existingObject) { + existingObject.setData(entry.objectData); + existingObject.setRegionalTimeserial(entry.regionalTimeserial); + continue; + } + + let newObject: LiveObject; + switch (entry.objectType) { + case 'LiveCounter': + newObject = new LiveCounter(this, entry.objectData, objectId); + break; + + case 'LiveMap': + newObject = new LiveMap(this, entry.objectData, objectId); + break; + + default: + throw new this._client.ErrorInfo(`Unknown live object type: ${entry.objectType}`, 40000, 400); + } + newObject.setRegionalTimeserial(entry.regionalTimeserial); + + this._liveObjectsPool.set(objectId, newObject); + } + + // need to remove LiveObject instances from the LiveObjectsPool for which objectIds were not received during the SYNC sequence + this._liveObjectsPool.deleteExtraObjectIds([...receivedObjectIds]); + } } diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index 3431992c1f..5f46c1a0ac 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -1,23 +1,46 @@ +import type BaseClient from 'common/lib/client/baseclient'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; import { LiveObjects } from './liveobjects'; -export type ObjectId = string; export const ROOT_OBJECT_ID = 'root'; +/** + * @internal + */ export class LiveObjectsPool { - private _pool: Map; + private _client: BaseClient; + private _pool: Map; constructor(private _liveObjects: LiveObjects) { + this._client = this._liveObjects.getClient(); this._pool = this._getInitialPool(); } - get(objectId: ObjectId): LiveObject | undefined { + get(objectId: string): LiveObject | undefined { return this._pool.get(objectId); } - private _getInitialPool(): Map { - const pool = new Map(); + /** + * Deletes objects from the pool for which object ids are not found in the provided array of ids. + */ + deleteExtraObjectIds(objectIds: string[]): void { + const poolObjectIds = [...this._pool.keys()]; + const extraObjectIds = this._client.Utils.arrSubtract(poolObjectIds, objectIds); + + extraObjectIds.forEach((x) => this._pool.delete(x)); + } + + set(objectId: string, liveObject: LiveObject): void { + this._pool.set(objectId, liveObject); + } + + reset(): void { + this._pool = this._getInitialPool(); + } + + private _getInitialPool(): Map { + const pool = new Map(); const root = new LiveMap(this._liveObjects, null, ROOT_OBJECT_ID); pool.set(root.getObjectId(), root); return pool; diff --git a/src/plugins/liveobjects/syncliveobjectsdatapool.ts b/src/plugins/liveobjects/syncliveobjectsdatapool.ts new file mode 100644 index 0000000000..1d30c5ad6a --- /dev/null +++ b/src/plugins/liveobjects/syncliveobjectsdatapool.ts @@ -0,0 +1,35 @@ +import { LiveObjectData } from './liveobject'; +import { LiveObjects } from './liveobjects'; + +export interface LiveObjectDataEntry { + objectData: LiveObjectData; + regionalTimeserial: string; + objectType: 'LiveMap' | 'LiveCounter'; +} + +/** + * @internal + */ +export class SyncLiveObjectsDataPool { + private _pool: Map; + + constructor(private _liveObjects: LiveObjects) { + this._pool = new Map(); + } + + entries() { + return this._pool.entries(); + } + + size(): number { + return this._pool.size; + } + + isEmpty(): boolean { + return this.size() === 0; + } + + reset(): void { + this._pool = new Map(); + } +} From d8755b31565227360450b47e937fdc520c6a549c Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 10 Oct 2024 08:09:19 +0100 Subject: [PATCH 020/166] Add missing properties to LiveMap --- src/plugins/liveobjects/livemap.ts | 24 ++++++++++++++----- src/plugins/liveobjects/liveobjects.ts | 8 ++++--- src/plugins/liveobjects/liveobjectspool.ts | 3 ++- .../liveobjects/syncliveobjectsdatapool.ts | 19 ++++++++++++--- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index b147dcf80f..ec30576df1 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -1,24 +1,27 @@ import { LiveObject, LiveObjectData } from './liveobject'; -import { StateValue } from './statemessage'; +import { LiveObjects } from './liveobjects'; +import { MapSemantics, StateValue } from './statemessage'; export interface ObjectIdStateData { - /** - * A reference to another state object, used to support composable state objects. - */ + /** A reference to another state object, used to support composable state objects. */ objectId: string; } export interface ValueStateData { /** - * A concrete leaf value in the state object graph. + * The encoding the client should use to interpret the value. + * Analogous to the `encoding` field on the `Message` and `PresenceMessage` types. */ + encoding?: string; + /** A concrete leaf value in the state object graph. */ value: StateValue; } export type StateData = ObjectIdStateData | ValueStateData; export interface MapEntry { - // TODO: add tombstone, timeserial + tombstone: boolean; + timeserial: string; data: StateData; } @@ -27,6 +30,15 @@ export interface LiveMapData extends LiveObjectData { } export class LiveMap extends LiveObject { + constructor( + liveObjects: LiveObjects, + private _semantics: MapSemantics, + initialData?: LiveMapData | null, + objectId?: string, + ) { + super(liveObjects, initialData, objectId); + } + /** * Returns the value associated with the specified key in the underlying Map object. * If no element is associated with the specified key, undefined is returned. diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index bc7ce74a61..c55d54b884 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -154,17 +154,19 @@ export class LiveObjects { } let newObject: LiveObject; - switch (entry.objectType) { + // assign to a variable so TS doesn't complain about 'never' type in the default case + const objectType = entry.objectType; + switch (objectType) { case 'LiveCounter': newObject = new LiveCounter(this, entry.objectData, objectId); break; case 'LiveMap': - newObject = new LiveMap(this, entry.objectData, objectId); + newObject = new LiveMap(this, entry.semantics, entry.objectData, objectId); break; default: - throw new this._client.ErrorInfo(`Unknown live object type: ${entry.objectType}`, 40000, 400); + throw new this._client.ErrorInfo(`Unknown live object type: ${objectType}`, 40000, 400); } newObject.setRegionalTimeserial(entry.regionalTimeserial); diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index 5f46c1a0ac..c19433caed 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -2,6 +2,7 @@ import type BaseClient from 'common/lib/client/baseclient'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; import { LiveObjects } from './liveobjects'; +import { MapSemantics } from './statemessage'; export const ROOT_OBJECT_ID = 'root'; @@ -41,7 +42,7 @@ export class LiveObjectsPool { private _getInitialPool(): Map { const pool = new Map(); - const root = new LiveMap(this._liveObjects, null, ROOT_OBJECT_ID); + const root = new LiveMap(this._liveObjects, MapSemantics.LWW, null, ROOT_OBJECT_ID); pool.set(root.getObjectId(), root); return pool; } diff --git a/src/plugins/liveobjects/syncliveobjectsdatapool.ts b/src/plugins/liveobjects/syncliveobjectsdatapool.ts index 1d30c5ad6a..3c787eb092 100644 --- a/src/plugins/liveobjects/syncliveobjectsdatapool.ts +++ b/src/plugins/liveobjects/syncliveobjectsdatapool.ts @@ -1,5 +1,6 @@ import { LiveObjectData } from './liveobject'; import { LiveObjects } from './liveobjects'; +import { MapSemantics } from './statemessage'; export interface LiveObjectDataEntry { objectData: LiveObjectData; @@ -7,14 +8,26 @@ export interface LiveObjectDataEntry { objectType: 'LiveMap' | 'LiveCounter'; } +export interface LiveCounterDataEntry extends LiveObjectDataEntry { + created: boolean; + objectType: 'LiveCounter'; +} + +export interface LiveMapDataEntry extends LiveObjectDataEntry { + objectType: 'LiveMap'; + semantics: MapSemantics; +} + +export type AnyDataEntry = LiveCounterDataEntry | LiveMapDataEntry; + /** * @internal */ export class SyncLiveObjectsDataPool { - private _pool: Map; + private _pool: Map; constructor(private _liveObjects: LiveObjects) { - this._pool = new Map(); + this._pool = new Map(); } entries() { @@ -30,6 +43,6 @@ export class SyncLiveObjectsDataPool { } reset(): void { - this._pool = new Map(); + this._pool = new Map(); } } From 1cc2bc274e2f27b77444e253efe246c68a5a6ea2 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 10 Oct 2024 08:11:18 +0100 Subject: [PATCH 021/166] Implement LiveObjects pool init from state SYNC sequence Resolves DTP-949 --- src/plugins/liveobjects/liveobjects.ts | 9 +- .../liveobjects/syncliveobjectsdatapool.ts | 90 ++++++++++++++++++- 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index c55d54b884..24833a719c 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -37,6 +37,13 @@ export class LiveObjects { return this._liveObjectsPool; } + /** + * @internal + */ + getChannel(): RealtimeChannel { + return this._channel; + } + /** * @internal */ @@ -53,7 +60,7 @@ export class LiveObjects { this._startNewSync(syncId, syncCursor); } - // TODO: delegate state messages to _syncLiveObjectsDataPool and create new live and data objects + this._syncLiveObjectsDataPool.applyStateMessages(stateMessages); // if this is the last (or only) message in a sequence of sync updates, end the sync if (!syncCursor) { diff --git a/src/plugins/liveobjects/syncliveobjectsdatapool.ts b/src/plugins/liveobjects/syncliveobjectsdatapool.ts index 3c787eb092..58de06ed6d 100644 --- a/src/plugins/liveobjects/syncliveobjectsdatapool.ts +++ b/src/plugins/liveobjects/syncliveobjectsdatapool.ts @@ -1,6 +1,10 @@ +import type BaseClient from 'common/lib/client/baseclient'; +import RealtimeChannel from 'common/lib/client/realtimechannel'; +import { LiveCounterData } from './livecounter'; +import { LiveMapData, MapEntry, ObjectIdStateData, StateData, ValueStateData } from './livemap'; import { LiveObjectData } from './liveobject'; import { LiveObjects } from './liveobjects'; -import { MapSemantics } from './statemessage'; +import { MapSemantics, StateMessage, StateObject } from './statemessage'; export interface LiveObjectDataEntry { objectData: LiveObjectData; @@ -24,9 +28,13 @@ export type AnyDataEntry = LiveCounterDataEntry | LiveMapDataEntry; * @internal */ export class SyncLiveObjectsDataPool { + private _client: BaseClient; + private _channel: RealtimeChannel; private _pool: Map; constructor(private _liveObjects: LiveObjects) { + this._client = this._liveObjects.getClient(); + this._channel = this._liveObjects.getChannel(); this._pool = new Map(); } @@ -45,4 +53,84 @@ export class SyncLiveObjectsDataPool { reset(): void { this._pool = new Map(); } + + applyStateMessages(stateMessages: StateMessage[]): void { + for (const stateMessage of stateMessages) { + if (!stateMessage.object) { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MAJOR, + 'LiveObjects.SyncLiveObjectsDataPool.applyStateMessages()', + `state message is received during SYNC without 'object' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + ); + continue; + } + + const stateObject = stateMessage.object; + + if (stateObject.counter) { + this._pool.set(stateObject.objectId, this._createLiveCounterDataEntry(stateObject)); + } else if (stateObject.map) { + this._pool.set(stateObject.objectId, this._createLiveMapDataEntry(stateObject)); + } else { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MINOR, + 'LiveObjects.SyncLiveObjectsDataPool.applyStateMessages()', + `received unsupported state object message during SYNC, expected 'counter' or 'map' to be present; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + ); + } + } + } + + private _createLiveCounterDataEntry(stateObject: StateObject): LiveCounterDataEntry { + const counter = stateObject.counter!; + + const objectData: LiveCounterData = { + data: counter.count ?? 0, + }; + const newEntry: LiveCounterDataEntry = { + created: counter.created, + objectData, + objectType: 'LiveCounter', + regionalTimeserial: stateObject.regionalTimeserial, + }; + + return newEntry; + } + + private _createLiveMapDataEntry(stateObject: StateObject): LiveMapDataEntry { + const map = stateObject.map!; + + const objectData: LiveMapData = { + data: new Map(), + }; + // need to iterate over entries manually to work around optional parameters from state object entries type + Object.entries(map.entries ?? {}).forEach(([key, entryFromMessage]) => { + let liveData: StateData; + if (typeof entryFromMessage.data.objectId !== 'undefined') { + liveData = { objectId: entryFromMessage.data.objectId } as ObjectIdStateData; + } else { + liveData = { encoding: entryFromMessage.data.encoding, value: entryFromMessage.data.value } as ValueStateData; + } + + const liveDataEntry: MapEntry = { + ...entryFromMessage, + // true only if we received explicit true. otherwise always false + tombstone: entryFromMessage.tombstone === true, + data: liveData, + }; + + objectData.data.set(key, liveDataEntry); + }); + + const newEntry: LiveMapDataEntry = { + objectData, + objectType: 'LiveMap', + regionalTimeserial: stateObject.regionalTimeserial, + semantics: map.semantics ?? MapSemantics.LWW, + }; + + return newEntry; + } } From 1d47d31d51052800e7b0e11006d9b413ff55c3f3 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 10 Oct 2024 08:20:50 +0100 Subject: [PATCH 022/166] Expose EventEmitter class on BaseClient to be used by plugins Plugins can't import EventEmitter class directly as that would increase the bundle size of a plugin --- src/common/lib/client/baseclient.ts | 2 ++ src/common/lib/client/modularplugins.ts | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/common/lib/client/baseclient.ts b/src/common/lib/client/baseclient.ts index 0b37e16176..be2cde2056 100644 --- a/src/common/lib/client/baseclient.ts +++ b/src/common/lib/client/baseclient.ts @@ -17,6 +17,7 @@ import { MsgPack } from 'common/types/msgpack'; import { HTTPRequestImplementations } from 'platform/web/lib/http/http'; import { FilteredSubscriptions } from './filteredsubscriptions'; import type { LocalDevice } from 'plugins/push/pushactivation'; +import EventEmitter from '../util/eventemitter'; type BatchResult = API.BatchResult; type BatchPublishSpec = API.BatchPublishSpec; @@ -179,6 +180,7 @@ class BaseClient { Logger = Logger; Defaults = Defaults; Utils = Utils; + EventEmitter = EventEmitter; } export default BaseClient; diff --git a/src/common/lib/client/modularplugins.ts b/src/common/lib/client/modularplugins.ts index e5a1736654..d51f03e530 100644 --- a/src/common/lib/client/modularplugins.ts +++ b/src/common/lib/client/modularplugins.ts @@ -10,8 +10,8 @@ import { fromValuesArray as presenceMessagesFromValuesArray, } from '../types/presencemessage'; import { TransportCtor } from '../transport/transport'; -import * as PushPlugin from 'plugins/push'; -import * as LiveObjectsPlugin from 'plugins/liveobjects'; +import type * as PushPlugin from 'plugins/push'; +import type * as LiveObjectsPlugin from 'plugins/liveobjects'; export interface PresenceMessagePlugin { presenceMessageFromValues: typeof presenceMessageFromValues; From 0e2c3a06436a318415db05da14d1eac966b12cb2 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 10 Oct 2024 08:36:41 +0100 Subject: [PATCH 023/166] Implement `LiveObjects.getRoot()` method Resolves DTP-951 --- src/plugins/liveobjects/liveobjects.ts | 23 +++++++++++++++++++++-- test/realtime/live_objects.test.js | 2 ++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 24833a719c..93b004206b 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -1,6 +1,7 @@ import type BaseClient from 'common/lib/client/baseclient'; import type RealtimeChannel from 'common/lib/client/realtimechannel'; import type * as API from '../../../ably'; +import type EventEmitter from 'common/lib/util/eventemitter'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; @@ -8,9 +9,16 @@ import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage } from './statemessage'; import { SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; +enum LiveObjectsEvents { + SyncCompleted = 'SyncCompleted', +} + export class LiveObjects { private _client: BaseClient; private _channel: RealtimeChannel; + // composition over inheritance since we cannot import class directly into plugin code. + // instead we obtain a class type from the client + private _eventEmitter: EventEmitter; private _liveObjectsPool: LiveObjectsPool; private _syncLiveObjectsDataPool: SyncLiveObjectsDataPool; private _syncInProgress: boolean; @@ -20,14 +28,24 @@ export class LiveObjects { constructor(channel: RealtimeChannel) { this._channel = channel; this._client = channel.client; + this._eventEmitter = new this._client.EventEmitter(this._client.logger); this._liveObjectsPool = new LiveObjectsPool(this); this._syncLiveObjectsDataPool = new SyncLiveObjectsDataPool(this); this._syncInProgress = true; } async getRoot(): Promise { - // TODO: wait for SYNC sequence to finish to return root - return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; + if (!this._syncInProgress) { + // SYNC is finished, can return immediately root object from pool + return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; + } + + // otherwise wait for SYNC sequence to finish + return new Promise((res) => { + this._eventEmitter.once(LiveObjectsEvents.SyncCompleted, () => { + res(this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap); + }); + }); } /** @@ -123,6 +141,7 @@ export class LiveObjects { this._currentSyncId = undefined; this._currentSyncCursor = undefined; this._syncInProgress = false; + this._eventEmitter.emit(LiveObjectsEvents.SyncCompleted); } private _parseSyncChannelSerial(syncChannelSerial: string | null | undefined): { diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index ad6ae38958..96acc5c7be 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -57,6 +57,7 @@ define(['ably', 'shared_helper', 'async', 'chai', 'live_objects'], function ( await helper.monitorConnectionThenCloseAndFinish(async () => { const channel = client.channels.get('channel'); const liveObjects = channel.liveObjects; + await channel.attach(); const root = await liveObjects.getRoot(); expect(root.constructor.name).to.equal('LiveMap'); @@ -71,6 +72,7 @@ define(['ably', 'shared_helper', 'async', 'chai', 'live_objects'], function ( await helper.monitorConnectionThenCloseAndFinish(async () => { const channel = client.channels.get('channel'); const liveObjects = channel.liveObjects; + await channel.attach(); const root = await liveObjects.getRoot(); helper.recordPrivateApi('call.LiveObject.getObjectId'); From d637bd7173f5644dfaed91c38e0f1a1c9d8fa47d Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 22 Oct 2024 06:24:56 +0100 Subject: [PATCH 024/166] Update `getRoot()` to use Promise version of `EventEmitter.once` Co-authored-by: Owen Pearson <48608556+owenpearson@users.noreply.github.com> --- src/plugins/liveobjects/liveobjects.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 93b004206b..e67ee350e3 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -35,17 +35,12 @@ export class LiveObjects { } async getRoot(): Promise { - if (!this._syncInProgress) { - // SYNC is finished, can return immediately root object from pool - return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; + // SYNC is currently in progress, wait for SYNC sequence to finish + if (this._syncInProgress) { + await this._eventEmitter.once(LiveObjectsEvents.SyncCompleted); } - // otherwise wait for SYNC sequence to finish - return new Promise((res) => { - this._eventEmitter.once(LiveObjectsEvents.SyncCompleted, () => { - res(this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap); - }); - }); + return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; } /** From 318fd76c716a91a0f0a4cdd244447551dd500cee Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 15 Oct 2024 09:27:41 +0100 Subject: [PATCH 025/166] Add LiveObjectsHelper module for tests --- test/common/globals/named_dependencies.js | 4 + test/common/modules/live_objects_helper.js | 171 +++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 test/common/modules/live_objects_helper.js diff --git a/test/common/globals/named_dependencies.js b/test/common/globals/named_dependencies.js index ead3217e7a..b303f0dfc9 100644 --- a/test/common/globals/named_dependencies.js +++ b/test/common/globals/named_dependencies.js @@ -26,5 +26,9 @@ define(function () { browser: 'test/common/modules/private_api_recorder', node: 'test/common/modules/private_api_recorder', }, + live_objects_helper: { + browser: 'test/common/modules/live_objects_helper', + node: 'test/common/modules/live_objects_helper', + }, }); }); diff --git a/test/common/modules/live_objects_helper.js b/test/common/modules/live_objects_helper.js new file mode 100644 index 0000000000..2f6ffc3843 --- /dev/null +++ b/test/common/modules/live_objects_helper.js @@ -0,0 +1,171 @@ +'use strict'; + +/** + * LiveObjects helper to create pre-determined state tree on channels + */ +define(['shared_helper'], function (Helper) { + const ACTIONS = { + MAP_CREATE: 0, + MAP_SET: 1, + MAP_REMOVE: 2, + COUNTER_CREATE: 3, + COUNTER_INC: 4, + }; + + function nonce() { + return Helper.randomString(); + } + + class LiveObjectsHelper { + /** + * Creates next LiveObjects state tree on a provided channel name: + * + * root "emptyMap" -> Map#1 {} -- empty map + * root "referencedMap" -> Map#2 { "counterKey": } + * root "valuesMap" -> Map#3 { "stringKey": "stringValue", "emptyStringKey": "", "bytesKey": , "emptyBytesKey": , "numberKey": 1, "zeroKey": 0, "trueKey": true, "falseKey": false, "mapKey": } + * root "emptyCounter" -> Counter#1 -- no initial value counter, should be 0 + * root "initialValueCounter" -> Counter#2 count=10 + * root "referencedCounter" -> Counter#3 count=20 + */ + async initForChannel(helper, channelName) { + const rest = helper.AblyRest({ useBinaryProtocol: false }); + + const emptyCounter = await this._createAndSetOnMap(rest, channelName, { + mapObjectId: 'root', + key: 'emptyCounter', + createOp: this._counterCreateOp(), + }); + const initialValueCounter = await this._createAndSetOnMap(rest, channelName, { + mapObjectId: 'root', + key: 'initialValueCounter', + createOp: this._counterCreateOp({ count: 10 }), + }); + const referencedCounter = await this._createAndSetOnMap(rest, channelName, { + mapObjectId: 'root', + key: 'referencedCounter', + createOp: this._counterCreateOp({ count: 20 }), + }); + + const emptyMap = await this._createAndSetOnMap(rest, channelName, { + mapObjectId: 'root', + key: 'emptyMap', + createOp: this._mapCreateOp(), + }); + const referencedMap = await this._createAndSetOnMap(rest, channelName, { + mapObjectId: 'root', + key: 'referencedMap', + createOp: this._mapCreateOp({ entries: { counterKey: { data: { objectId: referencedCounter.objectId } } } }), + }); + const valuesMap = await this._createAndSetOnMap(rest, channelName, { + mapObjectId: 'root', + key: 'valuesMap', + createOp: this._mapCreateOp({ + entries: { + stringKey: { data: { value: 'stringValue' } }, + emptyStringKey: { data: { value: '' } }, + bytesKey: { + data: { value: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9', encoding: 'base64' }, + }, + emptyBytesKey: { data: { value: '', encoding: 'base64' } }, + numberKey: { data: { value: 1 } }, + zeroKey: { data: { value: 0 } }, + trueKey: { data: { value: true } }, + falseKey: { data: { value: false } }, + mapKey: { data: { objectId: referencedMap.objectId } }, + }, + }), + }); + } + + async _createAndSetOnMap(rest, channelName, opts) { + const { mapObjectId, key, createOp } = opts; + + const createResult = await this._stateRequest(rest, channelName, createOp); + await this._stateRequest( + rest, + channelName, + this._mapSetOp({ objectId: mapObjectId, key, data: { objectId: createResult.objectId } }), + ); + + return createResult; + } + + _mapCreateOp(opts) { + const { objectId, entries } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.MAP_CREATE, + nonce: nonce(), + objectId, + }, + }; + + if (entries) { + op.operation.map = { entries }; + } + + return op; + } + + _mapSetOp(opts) { + const { objectId, key, data } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.MAP_SET, + objectId, + }, + }; + + if (key && data) { + op.operation.mapOp = { + key, + data, + }; + } + + return op; + } + + _counterCreateOp(opts) { + const { objectId, count } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.COUNTER_CREATE, + nonce: nonce(), + objectId, + }, + }; + + if (count != null) { + op.operation.counter = { count }; + } + + return op; + } + + async _stateRequest(rest, channelName, opBody) { + if (Array.isArray(opBody)) { + throw new Error(`Only single object state requests are supported`); + } + + const method = 'post'; + const path = `/channels/${channelName}/state`; + + const response = await rest.request(method, path, 3, null, opBody, null); + + if (response.success) { + // only one operation in request, so need only first item. + const result = response.items[0]; + // extract object id if present + result.objectId = result.objectIds?.[0]; + return result; + } + + throw new Error( + `${method}: ${path} FAILED; http code = ${response.statusCode}, error code = ${response.errorCode}, message = ${response.errorMessage}; operation = ${JSON.stringify(opBody)}`, + ); + } + } + + return (module.exports = new LiveObjectsHelper()); +}); From 45502e5f5cee038237cf6bbb4d015639eabcc6e7 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 15 Oct 2024 09:28:12 +0100 Subject: [PATCH 026/166] Enable `enableChannelState` feature flag for test apps --- test/common/modules/testapp_manager.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/common/modules/testapp_manager.js b/test/common/modules/testapp_manager.js index 0f4b7fa9dc..7852d79f19 100644 --- a/test/common/modules/testapp_manager.js +++ b/test/common/modules/testapp_manager.js @@ -133,6 +133,7 @@ define(['globals', 'ably'], function (ablyGlobals, ably) { callback(err); return; } + testData.post_apps.featureFlags = ['enableChannelState']; var postData = JSON.stringify(testData.post_apps); var postOptions = { host: restHost, From 948399930303c40658569e98868d2934030ac78b Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 17 Oct 2024 07:18:34 +0100 Subject: [PATCH 027/166] Add LiveMap.size() method --- src/plugins/liveobjects/livemap.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index ec30576df1..06cf160f4d 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -60,6 +60,10 @@ export class LiveMap extends LiveObject { } } + size(): number { + return this._dataRef.data.size; + } + protected _getZeroValueData(): LiveMapData { return { data: new Map() }; } From 687dee9575131409e34b5e07682ddd33e613bb21 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 17 Oct 2024 07:24:53 +0100 Subject: [PATCH 028/166] Refactor live objects test --- test/realtime/live_objects.test.js | 65 +++++++++++++++--------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 96acc5c7be..aabeca1421 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -1,16 +1,15 @@ 'use strict'; -define(['ably', 'shared_helper', 'async', 'chai', 'live_objects'], function ( +define(['ably', 'shared_helper', 'chai', 'live_objects'], function ( Ably, Helper, - async, chai, LiveObjectsPlugin, ) { - var expect = chai.expect; - var createPM = Ably.makeProtocolMessageFromDeserialized({ LiveObjectsPlugin }); + const expect = chai.expect; + const createPM = Ably.makeProtocolMessageFromDeserialized({ LiveObjectsPlugin }); - function LiveObjectsRealtime(helper, options) { + function RealtimeWithLiveObjects(helper, options) { return helper.AblyRealtime({ ...options, plugins: { LiveObjects: LiveObjectsPlugin } }); } @@ -41,44 +40,44 @@ define(['ably', 'shared_helper', 'async', 'chai', 'live_objects'], function ( describe('Realtime with LiveObjects plugin', () => { /** @nospec */ - it("returns LiveObjects instance when accessing channel's `liveObjects` property", async function () { + it("returns LiveObjects class instance when accessing channel's `liveObjects` property", async function () { const helper = this.test.helper; - const client = LiveObjectsRealtime(helper, { autoConnect: false }); + const client = RealtimeWithLiveObjects(helper, { autoConnect: false }); const channel = client.channels.get('channel'); expect(channel.liveObjects.constructor.name).to.equal('LiveObjects'); }); - describe('LiveObjects instance', () => { - /** @nospec */ - it('getRoot() returns LiveMap instance', async function () { - const helper = this.test.helper; - const client = LiveObjectsRealtime(helper); + /** @nospec */ + it('getRoot() returns LiveMap instance', async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper); - await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get('channel'); - const liveObjects = channel.liveObjects; - await channel.attach(); - const root = await liveObjects.getRoot(); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get('channel'); + const liveObjects = channel.liveObjects; - expect(root.constructor.name).to.equal('LiveMap'); - }, client); - }); + await channel.attach(); + const root = await liveObjects.getRoot(); + + expect(root.constructor.name).to.equal('LiveMap'); + }, client); + }); + + /** @nospec */ + it('getRoot() returns live object with id "root"', async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper); - /** @nospec */ - it('getRoot() returns live object with id "root"', async function () { - const helper = this.test.helper; - const client = LiveObjectsRealtime(helper); + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get('channel'); + const liveObjects = channel.liveObjects; - await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get('channel'); - const liveObjects = channel.liveObjects; - await channel.attach(); - const root = await liveObjects.getRoot(); + await channel.attach(); + const root = await liveObjects.getRoot(); - helper.recordPrivateApi('call.LiveObject.getObjectId'); - expect(root.getObjectId()).to.equal('root'); - }, client); - }); + helper.recordPrivateApi('call.LiveObject.getObjectId'); + expect(root.getObjectId()).to.equal('root'); + }, client); }); }); }); From 252678801e224566f87bffc352f20af2465cb742 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 17 Oct 2024 07:25:30 +0100 Subject: [PATCH 029/166] Add LiveObjectsHelper to live objects tests --- test/realtime/live_objects.test.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index aabeca1421..dd3dba2d5d 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -1,13 +1,15 @@ 'use strict'; -define(['ably', 'shared_helper', 'chai', 'live_objects'], function ( +define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], function ( Ably, Helper, chai, LiveObjectsPlugin, + LiveObjectsHelper, ) { const expect = chai.expect; const createPM = Ably.makeProtocolMessageFromDeserialized({ LiveObjectsPlugin }); + const liveObjectsFixturesChannel = 'liveobjects_fixtures'; function RealtimeWithLiveObjects(helper, options) { return helper.AblyRealtime({ ...options, plugins: { LiveObjects: LiveObjectsPlugin } }); @@ -24,7 +26,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects'], function ( done(err); return; } - done(); + + LiveObjectsHelper.initForChannel(helper, liveObjectsFixturesChannel) + .then(done) + .catch((err) => done(err)); }); }); From 7faa72c443ce1e627da58af178137eba4b4c068d Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 17 Oct 2024 07:28:53 +0100 Subject: [PATCH 030/166] Add more getRoot() tests for LiveObjects --- test/realtime/live_objects.test.js | 54 ++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index dd3dba2d5d..71c53fc2e2 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -10,11 +10,19 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const expect = chai.expect; const createPM = Ably.makeProtocolMessageFromDeserialized({ LiveObjectsPlugin }); const liveObjectsFixturesChannel = 'liveobjects_fixtures'; + const nextTick = Ably.Realtime.Platform.Config.nextTick; function RealtimeWithLiveObjects(helper, options) { return helper.AblyRealtime({ ...options, plugins: { LiveObjects: LiveObjectsPlugin } }); } + function channelOptionsWithLiveObjects(options) { + return { + ...options, + modes: ['STATE_SUBSCRIBE', 'STATE_PUBLISH'], + }; + } + describe('realtime/live_objects', function () { this.timeout(60 * 1000); @@ -52,13 +60,29 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(channel.liveObjects.constructor.name).to.equal('LiveObjects'); }); + /** @nospec */ + it('getRoot() returns empty root when no state exist on a channel', async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + expect(root.size()).to.equal(0, 'Check root has no keys'); + }, client); + }); + /** @nospec */ it('getRoot() returns LiveMap instance', async function () { const helper = this.test.helper; const client = RealtimeWithLiveObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get('channel'); + const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); const liveObjects = channel.liveObjects; await channel.attach(); @@ -74,7 +98,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const client = RealtimeWithLiveObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get('channel'); + const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); const liveObjects = channel.liveObjects; await channel.attach(); @@ -84,6 +108,32 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(root.getObjectId()).to.equal('root'); }, client); }); + + /** @nospec */ + it('getRoot() resolves immediately when STATE_SYNC sequence is completed', async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + // wait for STATE_SYNC sequence to complete by accessing root for the first time + await liveObjects.getRoot(); + + let resolvedImmediately = false; + liveObjects.getRoot().then(() => { + resolvedImmediately = true; + }); + + // wait for next tick for getRoot() handler to process + helper.recordPrivateApi('call.Platform.nextTick'); + await new Promise((res) => nextTick(res)); + + expect(resolvedImmediately, 'Check getRoot() is resolved on next tick').to.be.true; + }, client); + }); }); }); }); From 7d4bcd997b3913b90470bb15f9f8896845481595 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 17 Oct 2024 07:30:42 +0100 Subject: [PATCH 031/166] Add STATE_SYNC ProtocolMessage test when LiveObjects plugin is not provided --- test/realtime/live_objects.test.js | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 71c53fc2e2..339b0460fb 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -49,6 +49,49 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const channel = client.channels.get('channel'); expect(() => channel.liveObjects).to.throw('LiveObjects plugin not provided'); }); + + /** @nospec */ + it('doesn’t break when it receives a STATE_SYNC ProtocolMessage', async function () { + const helper = this.test.helper; + const testClient = helper.AblyRealtime(); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const testChannel = testClient.channels.get('channel'); + await testChannel.attach(); + + const receivedMessagePromise = new Promise((resolve) => testChannel.subscribe(resolve)); + + const publishClient = helper.AblyRealtime(); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + // inject STATE_SYNC message that should be ignored and not break anything without LiveObjects plugin + helper.recordPrivateApi('call.channel.processMessage'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + await testChannel.processMessage( + createPM({ + action: 20, + channel: 'channel', + channelSerial: 'serial:', + state: [ + { + object: { + objectId: 'root', + regionalTimeserial: '@0-0', + map: {}, + }, + }, + ], + }), + ); + + const publishChannel = publishClient.channels.get('channel'); + await publishChannel.publish(null, 'test'); + + // regular message subscriptions should still work after processing STATE_SYNC message without LiveObjects plugin + await receivedMessagePromise; + }, publishClient); + }, testClient); + }); }); describe('Realtime with LiveObjects plugin', () => { From a0c3b6cbeee5316450812c095e2488d1f14fe85c Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 17 Oct 2024 07:38:01 +0100 Subject: [PATCH 032/166] Add initial STATE_SYNC sequence handling tests --- test/realtime/live_objects.test.js | 162 +++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 339b0460fb..d6d34ac47d 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -8,6 +8,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], LiveObjectsHelper, ) { const expect = chai.expect; + const BufferUtils = Ably.Realtime.Platform.BufferUtils; const createPM = Ably.makeProtocolMessageFromDeserialized({ LiveObjectsPlugin }); const liveObjectsFixturesChannel = 'liveobjects_fixtures'; const nextTick = Ably.Realtime.Platform.Config.nextTick; @@ -177,6 +178,167 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(resolvedImmediately, 'Check getRoot() is resolved on next tick').to.be.true; }, client); }); + + it('builds state object tree from STATE_SYNC sequence on channel attachment', async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + const counterKeys = ['emptyCounter', 'initialValueCounter', 'referencedCounter']; + const mapKeys = ['emptyMap', 'referencedMap', 'valuesMap']; + const rootKeysCount = counterKeys.length + mapKeys.length; + + expect(root, 'Check getRoot() is resolved when STATE_SYNC sequence ends').to.exist; + expect(root.size()).to.equal(rootKeysCount, 'Check root has correct number of keys'); + + counterKeys.forEach((key) => { + const counter = root.get(key); + expect(counter, `Check counter at key="${key}" in root exists`).to.exist; + expect(counter.constructor.name).to.equal( + 'LiveCounter', + `Check counter at key="${key}" in root is of type LiveCounter`, + ); + }); + + mapKeys.forEach((key) => { + const map = root.get(key); + expect(map, `Check map at key="${key}" in root exists`).to.exist; + expect(map.constructor.name).to.equal('LiveMap', `Check map at key="${key}" in root is of type LiveMap`); + }); + + const valuesMap = root.get('valuesMap'); + const valueMapKeys = [ + 'stringKey', + 'emptyStringKey', + 'bytesKey', + 'emptyBytesKey', + 'numberKey', + 'zeroKey', + 'trueKey', + 'falseKey', + 'mapKey', + ]; + expect(valuesMap.size()).to.equal(valueMapKeys.length, 'Check nested map has correct number of keys'); + valueMapKeys.forEach((key) => { + const value = valuesMap.get(key); + expect(value, `Check value at key="${key}" in nested map exists`).to.exist; + }); + }, client); + }); + + it('LiveCounter is initialized with initial value from STATE_SYNC sequence', async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + const counters = [ + { key: 'emptyCounter', value: 0 }, + { key: 'initialValueCounter', value: 10 }, + { key: 'referencedCounter', value: 20 }, + ]; + + counters.forEach((x) => { + const counter = root.get(x.key); + expect(counter.value()).to.equal(x.value, `Check counter at key="${x.key}" in root has correct value`); + }); + }, client); + }); + + it('LiveMap is initialized with initial value from STATE_SYNC sequence', async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + const emptyMap = root.get('emptyMap'); + expect(emptyMap.size()).to.equal(0, 'Check empty map in root has no keys'); + + const referencedMap = root.get('referencedMap'); + expect(referencedMap.size()).to.equal(1, 'Check referenced map in root has correct number of keys'); + + const counterFromReferencedMap = referencedMap.get('counterKey'); + expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); + + const valuesMap = root.get('valuesMap'); + expect(valuesMap.size()).to.equal(9, 'Check values map in root has correct number of keys'); + + expect(valuesMap.get('stringKey')).to.equal('stringValue', 'Check values map has correct string value key'); + expect(valuesMap.get('emptyStringKey')).to.equal('', 'Check values map has correct empty string value key'); + expect( + BufferUtils.areBuffersEqual( + valuesMap.get('bytesKey'), + BufferUtils.base64Decode('eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9'), + ), + 'Check values map has correct bytes value key', + ).to.be.true; + expect( + BufferUtils.areBuffersEqual(valuesMap.get('emptyBytesKey'), BufferUtils.base64Decode('')), + 'Check values map has correct empty bytes value key', + ).to.be.true; + expect(valuesMap.get('numberKey')).to.equal(1, 'Check values map has correct number value key'); + expect(valuesMap.get('zeroKey')).to.equal(0, 'Check values map has correct zero number value key'); + expect(valuesMap.get('trueKey')).to.equal(true, `Check values map has correct 'true' value key`); + expect(valuesMap.get('falseKey')).to.equal(false, `Check values map has correct 'false' value key`); + + const mapFromValuesMap = valuesMap.get('mapKey'); + expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); + }, client); + }); + + it('LiveMaps can reference the same object in their keys', async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + const referencedCounter = root.get('referencedCounter'); + const referencedMap = root.get('referencedMap'); + const valuesMap = root.get('valuesMap'); + + const counterFromReferencedMap = referencedMap.get('counterKey'); + expect(counterFromReferencedMap, 'Check nested counter exists at a key in a map').to.exist; + expect(counterFromReferencedMap.constructor.name).to.equal( + 'LiveCounter', + 'Check nested counter is of type LiveCounter', + ); + expect(counterFromReferencedMap).to.equal( + referencedCounter, + 'Check nested counter is the same object instance as counter on the root', + ); + expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); + + const mapFromValuesMap = valuesMap.get('mapKey'); + expect(mapFromValuesMap, 'Check nested map exists at a key in a map').to.exist; + expect(mapFromValuesMap.constructor.name).to.equal('LiveMap', 'Check nested map is of type LiveMap'); + expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); + expect(mapFromValuesMap).to.equal( + referencedMap, + 'Check nested map is the same object instance as map on the root', + ); + }, client); + }); }); }); }); From 3206fe01cbf0bdc5760769ca53d7a15cb635c634 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 17 Oct 2024 07:57:49 +0100 Subject: [PATCH 033/166] Add test getRoot() while STATE_SYNC is in progress --- test/realtime/live_objects.test.js | 120 +++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 7 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index d6d34ac47d..d2dbc39565 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -105,7 +105,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); /** @nospec */ - it('getRoot() returns empty root when no state exist on a channel', async function () { + it('getRoot() returns LiveMap instance', async function () { const helper = this.test.helper; const client = RealtimeWithLiveObjects(helper); @@ -116,12 +116,12 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await channel.attach(); const root = await liveObjects.getRoot(); - expect(root.size()).to.equal(0, 'Check root has no keys'); + expect(root.constructor.name).to.equal('LiveMap'); }, client); }); /** @nospec */ - it('getRoot() returns LiveMap instance', async function () { + it('getRoot() returns live object with id "root"', async function () { const helper = this.test.helper; const client = RealtimeWithLiveObjects(helper); @@ -132,12 +132,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await channel.attach(); const root = await liveObjects.getRoot(); - expect(root.constructor.name).to.equal('LiveMap'); + helper.recordPrivateApi('call.LiveObject.getObjectId'); + expect(root.getObjectId()).to.equal('root'); }, client); }); /** @nospec */ - it('getRoot() returns live object with id "root"', async function () { + it('getRoot() returns empty root when no state exist on a channel', async function () { const helper = this.test.helper; const client = RealtimeWithLiveObjects(helper); @@ -148,8 +149,35 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await channel.attach(); const root = await liveObjects.getRoot(); - helper.recordPrivateApi('call.LiveObject.getObjectId'); - expect(root.getObjectId()).to.equal('root'); + expect(root.size()).to.equal(0, 'Check root has no keys'); + }, client); + }); + + /** @nospec */ + it('getRoot() waits for initial STATE_SYNC to be completed before resolving', async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + const getRootPromise = liveObjects.getRoot(); + + let getRootResolved = false; + getRootPromise.then(() => { + getRootResolved = true; + }); + + // give a chance for getRoot() to resolve and proc its handler. it should not + helper.recordPrivateApi('call.Platform.nextTick'); + await new Promise((res) => nextTick(res)); + expect(getRootResolved, 'Check getRoot() is not resolved until STATE_SYNC sequence is completed').to.be.false; + + await channel.attach(); + + // should resolve eventually after attach + await getRootPromise; }, client); }); @@ -179,6 +207,84 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, client); }); + /** @nospec */ + it('getRoot() waits for subsequent STATE_SYNC to finish before resolving', async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + // wait for initial STATE_SYNC sequence to complete + await liveObjects.getRoot(); + + // inject STATE_SYNC message to emulate start of new sequence + helper.recordPrivateApi('call.channel.processMessage'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + await channel.processMessage( + createPM({ + action: 20, + channel: 'channel', + // have cursor so client awaits for additional STATE_SYNC messages + channelSerial: 'serial:cursor', + state: [], + }), + ); + + let getRootResolved = false; + let newRoot; + liveObjects.getRoot().then((value) => { + getRootResolved = true; + newRoot = value; + }); + + // wait for next tick to check that getRoot() promise handler didn't proc + helper.recordPrivateApi('call.Platform.nextTick'); + await new Promise((res) => nextTick(res)); + + expect(getRootResolved, 'Check getRoot() is not resolved while STATE_SYNC is in progress').to.be.false; + + // inject next STATE_SYNC message + helper.recordPrivateApi('call.channel.processMessage'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + await channel.processMessage( + createPM({ + action: 20, + channel: 'channel', + // no cursor to indicate the end of STATE_SYNC messages + channelSerial: 'serial:', + state: [ + { + object: { + objectId: 'root', + regionalTimeserial: '@0-0', + map: { + entries: { + key: { + timeserial: '@0-0', + data: { + value: 1, + }, + }, + }, + }, + }, + }, + ], + }), + ); + + // wait for next tick for getRoot() handler to process + helper.recordPrivateApi('call.Platform.nextTick'); + await new Promise((res) => nextTick(res)); + + expect(getRootResolved, 'Check getRoot() is resolved when STATE_SYNC sequence has ended').to.be.true; + expect(newRoot.get('key')).to.equal(1, 'Check new root after STATE_SYNC sequence has expected key'); + }, client); + }); + it('builds state object tree from STATE_SYNC sequence on channel attachment', async function () { const helper = this.test.helper; const client = RealtimeWithLiveObjects(helper); From 566bf00848824531a01adf4a96d9fae0a8b243e1 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 17 Oct 2024 08:44:38 +0100 Subject: [PATCH 034/166] Add test for LiveObjects state modes --- test/realtime/live_objects.test.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index d2dbc39565..e3c0d078ea 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -446,5 +446,22 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, client); }); }); + + it('can attach to channel with LiveObjects state modes', async function () { + const helper = this.test.helper; + const client = helper.AblyRealtime(); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const liveObjectsModes = ['state_subscribe', 'state_publish']; + const channelOptions = { modes: liveObjectsModes }; + const channel = client.channels.get('channel', channelOptions); + + await channel.attach(); + + helper.recordPrivateApi('read.channel.channelOptions'); + expect(channel.channelOptions).to.deep.equal(channelOptions, 'Check expected channel options'); + expect(channel.modes).to.deep.equal(liveObjectsModes, 'Check expected modes'); + }, client); + }); }); }); From c3a15ac5f09169c19afe1e5371c66b6cb8b4c24d Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 22 Oct 2024 10:00:41 +0100 Subject: [PATCH 035/166] Minor changes for slightly better performance and readability --- src/plugins/liveobjects/liveobjectspool.ts | 2 +- src/plugins/liveobjects/syncliveobjectsdatapool.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index c19433caed..f4345d61e9 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -27,7 +27,7 @@ export class LiveObjectsPool { */ deleteExtraObjectIds(objectIds: string[]): void { const poolObjectIds = [...this._pool.keys()]; - const extraObjectIds = this._client.Utils.arrSubtract(poolObjectIds, objectIds); + const extraObjectIds = poolObjectIds.filter((x) => !objectIds.includes(x)); extraObjectIds.forEach((x) => this._pool.delete(x)); } diff --git a/src/plugins/liveobjects/syncliveobjectsdatapool.ts b/src/plugins/liveobjects/syncliveobjectsdatapool.ts index 58de06ed6d..73de256ff5 100644 --- a/src/plugins/liveobjects/syncliveobjectsdatapool.ts +++ b/src/plugins/liveobjects/syncliveobjectsdatapool.ts @@ -47,11 +47,11 @@ export class SyncLiveObjectsDataPool { } isEmpty(): boolean { - return this.size() === 0; + return this._pool.size === 0; } reset(): void { - this._pool = new Map(); + this._pool.clear(); } applyStateMessages(stateMessages: StateMessage[]): void { From fbf9d422195aad85b291d1f46db010a9853742e9 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 22 Oct 2024 10:01:32 +0100 Subject: [PATCH 036/166] Fix `decodeMapEntry` call was not awaited in `StateMessage.decode` call --- src/plugins/liveobjects/statemessage.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index c8063e4550..858f87ea1c 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -155,13 +155,13 @@ export class StateMessage { if (message.object?.map?.entries) { for (const entry of Object.values(message.object.map.entries)) { - decodeMapEntry(entry, inputContext, decodeDataFn); + await decodeMapEntry(entry, inputContext, decodeDataFn); } } if (message.operation?.map) { for (const entry of Object.values(message.operation.map)) { - decodeMapEntry(entry, inputContext, decodeDataFn); + await decodeMapEntry(entry, inputContext, decodeDataFn); } } From b73a15cf34b876ba4c6f1375c5f425cea18f6dbc Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 25 Oct 2024 07:35:45 +0100 Subject: [PATCH 037/166] Remove Utils.arrSubtract, as it is preffered to use `arr.filter((x) => !otherArr.includes(x))` instead --- src/common/lib/util/utils.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/common/lib/util/utils.ts b/src/common/lib/util/utils.ts index f1af5ff363..b88f9dd4b3 100644 --- a/src/common/lib/util/utils.ts +++ b/src/common/lib/util/utils.ts @@ -165,15 +165,6 @@ export function arrIntersectOb(arr: Array, ob: Partial(arr1: Array, arr2: Array): Array { - const result = []; - for (let i = 0; i < arr1.length; i++) { - const element = arr1[i]; - if (arr2.indexOf(element) == -1) result.push(element); - } - return result; -} - export function arrDeleteValue(arr: Array, val: T): boolean { const idx = arr.indexOf(val); const res = idx != -1; From 8e31a6a6d702213804fbe32f109226850ea3198a Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 18 Oct 2024 06:46:10 +0100 Subject: [PATCH 038/166] Add Timeserial class to LiveObjects plugin This is a copy of Timeserial implementation from chat-js repo [1], with some changes: - inversion of control change to pass in the reference to Ably BaseClient, as we cannot import it directly in a plugin - seriesId comparison fix for empty seriesId. This is based on the identical fix in realtime [2] - slightly better error handling [1] https://github.com/ably/ably-chat-js/blob/main/src/core/timeserial.ts [2] https://github.com/ably/realtime/pull/6678/files#diff-31896aac1d68683fb340b9ac488b1cfd5b96eb9c0b79f2260e637c5004721e98R134-R143 --- src/plugins/liveobjects/timeserial.ts | 179 ++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 src/plugins/liveobjects/timeserial.ts diff --git a/src/plugins/liveobjects/timeserial.ts b/src/plugins/liveobjects/timeserial.ts new file mode 100644 index 0000000000..553f960a50 --- /dev/null +++ b/src/plugins/liveobjects/timeserial.ts @@ -0,0 +1,179 @@ +import type BaseClient from 'common/lib/client/baseclient'; + +/** + * Represents a parsed timeserial. + */ +export interface Timeserial { + /** + * The series ID of the timeserial. + */ + readonly seriesId: string; + + /** + * The timestamp of the timeserial. + */ + readonly timestamp: number; + + /** + * The counter of the timeserial. + */ + readonly counter: number; + + /** + * The index of the timeserial. + */ + readonly index?: number; + + toString(): string; + + before(timeserial: Timeserial | string): boolean; + + after(timeserial: Timeserial | string): boolean; + + equal(timeserial: Timeserial | string): boolean; +} + +/** + * Default implementation of the Timeserial interface. Used internally to parse and compare timeserials. + * + * @internal + */ +export class DefaultTimeserial implements Timeserial { + public readonly seriesId: string; + public readonly timestamp: number; + public readonly counter: number; + public readonly index?: number; + + private constructor( + private _client: BaseClient, + seriesId: string, + timestamp: number, + counter: number, + index?: number, + ) { + this.seriesId = seriesId; + this.timestamp = timestamp; + this.counter = counter; + this.index = index; + } + + /** + * Returns the string representation of the timeserial object. + * @returns The timeserial string. + */ + toString(): string { + return `${this.seriesId}@${this.timestamp.toString()}-${this.counter.toString()}${this.index ? `:${this.index.toString()}` : ''}`; + } + + /** + * Calculate the timeserial object from a timeserial string. + * + * @param timeserial The timeserial string to parse. + * @returns The parsed timeserial object. + * @throws {@link BaseClient.ErrorInfo | ErrorInfo} if timeserial is invalid. + */ + static calculateTimeserial(client: BaseClient, timeserial: string | null | undefined): Timeserial { + if (client.Utils.isNil(timeserial)) { + throw new client.ErrorInfo(`Invalid timeserial: ${timeserial}`, 50000, 500); + } + + const [seriesId, rest] = timeserial.split('@'); + if (!seriesId || !rest) { + throw new client.ErrorInfo(`Invalid timeserial: ${timeserial}`, 50000, 500); + } + + const [timestamp, counterAndIndex] = rest.split('-'); + if (!timestamp || !counterAndIndex) { + throw new client.ErrorInfo(`Invalid timeserial: ${timeserial}`, 50000, 500); + } + + const [counter, index] = counterAndIndex.split(':'); + if (!counter) { + throw new client.ErrorInfo(`Invalid timeserial: ${timeserial}`, 50000, 500); + } + + return new DefaultTimeserial( + client, + seriesId, + Number(timestamp), + Number(counter), + index ? Number(index) : undefined, + ); + } + + /** + * Compares this timeserial to the supplied timeserial, returning a number indicating their relative order. + * @param timeserialToCompare The timeserial to compare against. Can be a string or a Timeserial object. + * @returns 0 if the timeserials are equal, <0 if the first timeserial is less than the second, >0 if the first timeserial is greater than the second. + * @throws {@link BaseClient.ErrorInfo | ErrorInfo} if comparison timeserial is invalid. + */ + private _timeserialCompare(timeserialToCompare: string | Timeserial): number { + const secondTimeserial = + typeof timeserialToCompare === 'string' + ? DefaultTimeserial.calculateTimeserial(this._client, timeserialToCompare) + : timeserialToCompare; + + // Compare the timestamp + const timestampDiff = this.timestamp - secondTimeserial.timestamp; + if (timestampDiff) { + return timestampDiff; + } + + // Compare the counter + const counterDiff = this.counter - secondTimeserial.counter; + if (counterDiff) { + return counterDiff; + } + + // Compare the seriesId + // An empty seriesId is considered less than a non-empty one + if (!this.seriesId && secondTimeserial.seriesId) { + return -1; + } + if (this.seriesId && !secondTimeserial.seriesId) { + return 1; + } + // Otherwise compare seriesId lexicographically + const seriesIdDiff = + this.seriesId === secondTimeserial.seriesId ? 0 : this.seriesId < secondTimeserial.seriesId ? -1 : 1; + + if (seriesIdDiff) { + return seriesIdDiff; + } + + // Compare the index, if present + return this.index !== undefined && secondTimeserial.index !== undefined ? this.index - secondTimeserial.index : 0; + } + + /** + * Determines if this timeserial occurs logically before the given timeserial. + * + * @param timeserial The timeserial to compare against. Can be a string or a Timeserial object. + * @returns true if this timeserial precedes the given timeserial, in global order. + * @throws {@link BaseClient.ErrorInfo | ErrorInfo} if the given timeserial is invalid. + */ + before(timeserial: Timeserial | string): boolean { + return this._timeserialCompare(timeserial) < 0; + } + + /** + * Determines if this timeserial occurs logically after the given timeserial. + * + * @param timeserial The timeserial to compare against. Can be a string or a Timeserial object. + * @returns true if this timeserial follows the given timeserial, in global order. + * @throws {@link BaseClient.ErrorInfo | ErrorInfo} if the given timeserial is invalid. + */ + after(timeserial: Timeserial | string): boolean { + return this._timeserialCompare(timeserial) > 0; + } + + /** + * Determines if this timeserial is equal to the given timeserial. + * @param timeserial The timeserial to compare against. Can be a string or a Timeserial object. + * @returns true if this timeserial is equal to the given timeserial. + * @throws {@link BaseClient.ErrorInfo | ErrorInfo} if the given timeserial is invalid. + */ + equal(timeserial: Timeserial | string): boolean { + return this._timeserialCompare(timeserial) === 0; + } +} From 692f6eddd689d37e7d994c4f7ada784930f2a0ef Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 18 Oct 2024 08:23:20 +0100 Subject: [PATCH 039/166] Add ObjectId class to parse object id strings --- src/plugins/liveobjects/objectid.ts | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 src/plugins/liveobjects/objectid.ts diff --git a/src/plugins/liveobjects/objectid.ts b/src/plugins/liveobjects/objectid.ts new file mode 100644 index 0000000000..c968eefd60 --- /dev/null +++ b/src/plugins/liveobjects/objectid.ts @@ -0,0 +1,35 @@ +import type BaseClient from 'common/lib/client/baseclient'; + +export type LiveObjectType = 'map' | 'counter'; + +/** + * Represents a parsed object id. + * + * @internal + */ +export class ObjectId { + private constructor( + readonly type: LiveObjectType, + readonly hash: string, + ) {} + + /** + * Create ObjectId instance from hashed object id string. + */ + static fromString(client: BaseClient, objectId: string | null | undefined): ObjectId { + if (client.Utils.isNil(objectId)) { + throw new client.ErrorInfo('Invalid object id string', 50000, 500); + } + + const [type, hash] = objectId.split(':'); + if (!type || !hash) { + throw new client.ErrorInfo('Invalid object id string', 50000, 500); + } + + if (!['map', 'counter'].includes(type)) { + throw new client.ErrorInfo(`Invalid object type in object id: ${objectId}`, 50000, 500); + } + + return new ObjectId(type as LiveObjectType, hash); + } +} From 131fc4e1a97aea5ea8ed07f3d232a06a87ad6915 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 18 Oct 2024 08:44:15 +0100 Subject: [PATCH 040/166] Preparation for CRDT operations implementation - Add handling of `created` field to LiveCounter - Add handling of `tombstone` field on entries to LiveMap - Change `timeserial` in LiveMap entries to be of type Timeserial - Change `data` in LiveMap entries to be optionally undefined --- src/plugins/liveobjects/livecounter.ts | 24 +++++++++++++++ src/plugins/liveobjects/livemap.ts | 30 +++++++++++++++---- src/plugins/liveobjects/liveobjects.ts | 7 +++-- .../liveobjects/syncliveobjectsdatapool.ts | 4 ++- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 06398d6e9f..48dfa5365f 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -1,14 +1,38 @@ import { LiveObject, LiveObjectData } from './liveobject'; +import { LiveObjects } from './liveobjects'; export interface LiveCounterData extends LiveObjectData { data: number; } export class LiveCounter extends LiveObject { + constructor( + liveObjects: LiveObjects, + private _created: boolean, + initialData?: LiveCounterData | null, + objectId?: string, + ) { + super(liveObjects, initialData, objectId); + } + value(): number { return this._dataRef.data; } + /** + * @internal + */ + isCreated(): boolean { + return this._created; + } + + /** + * @internal + */ + setCreated(created: boolean): void { + this._created = created; + } + protected _getZeroValueData(): LiveCounterData { return { data: 0 }; } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 06cf160f4d..8c13b6d403 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -1,6 +1,7 @@ import { LiveObject, LiveObjectData } from './liveobject'; import { LiveObjects } from './liveobjects'; import { MapSemantics, StateValue } from './statemessage'; +import { Timeserial } from './timeserial'; export interface ObjectIdStateData { /** A reference to another state object, used to support composable state objects. */ @@ -21,8 +22,8 @@ export type StateData = ObjectIdStateData | ValueStateData; export interface MapEntry { tombstone: boolean; - timeserial: string; - data: StateData; + timeserial: Timeserial; + data: StateData | undefined; } export interface LiveMapData extends LiveObjectData { @@ -53,15 +54,32 @@ export class LiveMap extends LiveObject { return undefined; } - if ('value' in element.data) { - return element.data.value; + if (element.tombstone === true) { + return undefined; + } + + // data exists for non-tombstoned elements + const data = element.data!; + + if ('value' in data) { + return data.value; } else { - return this._liveObjects.getPool().get(element.data.objectId); + return this._liveObjects.getPool().get(data.objectId); } } size(): number { - return this._dataRef.data.size; + let size = 0; + for (const value of this._dataRef.data.values()) { + if (value.tombstone === true) { + // should not count deleted entries + continue; + } + + size++; + } + + return size; } protected _getZeroValueData(): LiveMapData { diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index e67ee350e3..11a5ef3a69 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -7,7 +7,7 @@ import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage } from './statemessage'; -import { SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; +import { LiveCounterDataEntry, SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; enum LiveObjectsEvents { SyncCompleted = 'SyncCompleted', @@ -171,6 +171,9 @@ export class LiveObjects { if (existingObject) { existingObject.setData(entry.objectData); existingObject.setRegionalTimeserial(entry.regionalTimeserial); + if (existingObject instanceof LiveCounter) { + existingObject.setCreated((entry as LiveCounterDataEntry).created); + } continue; } @@ -179,7 +182,7 @@ export class LiveObjects { const objectType = entry.objectType; switch (objectType) { case 'LiveCounter': - newObject = new LiveCounter(this, entry.objectData, objectId); + newObject = new LiveCounter(this, entry.created, entry.objectData, objectId); break; case 'LiveMap': diff --git a/src/plugins/liveobjects/syncliveobjectsdatapool.ts b/src/plugins/liveobjects/syncliveobjectsdatapool.ts index 73de256ff5..22e83c7a19 100644 --- a/src/plugins/liveobjects/syncliveobjectsdatapool.ts +++ b/src/plugins/liveobjects/syncliveobjectsdatapool.ts @@ -5,6 +5,7 @@ import { LiveMapData, MapEntry, ObjectIdStateData, StateData, ValueStateData } f import { LiveObjectData } from './liveobject'; import { LiveObjects } from './liveobjects'; import { MapSemantics, StateMessage, StateObject } from './statemessage'; +import { DefaultTimeserial } from './timeserial'; export interface LiveObjectDataEntry { objectData: LiveObjectData; @@ -90,10 +91,10 @@ export class SyncLiveObjectsDataPool { data: counter.count ?? 0, }; const newEntry: LiveCounterDataEntry = { - created: counter.created, objectData, objectType: 'LiveCounter', regionalTimeserial: stateObject.regionalTimeserial, + created: counter.created, }; return newEntry; @@ -116,6 +117,7 @@ export class SyncLiveObjectsDataPool { const liveDataEntry: MapEntry = { ...entryFromMessage, + timeserial: DefaultTimeserial.calculateTimeserial(this._client, entryFromMessage.timeserial), // true only if we received explicit true. otherwise always false tombstone: entryFromMessage.tombstone === true, data: liveData, From 9b83e941c00ea92db4934b7f34d883ef4bb311c0 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 18 Oct 2024 06:18:55 +0100 Subject: [PATCH 041/166] Implement support for applying incoming operations to LiveMap/LiveCounter This adds implementation for CRDT operations for LiveMap/LiveCounter classes to be able to handle incoming state operation messages. Resolves DTP-954 --- src/plugins/liveobjects/livecounter.ts | 73 +++++++++ src/plugins/liveobjects/livemap.ts | 170 ++++++++++++++++++++- src/plugins/liveobjects/liveobject.ts | 8 + src/plugins/liveobjects/liveobjectspool.ts | 23 +++ 4 files changed, 272 insertions(+), 2 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 48dfa5365f..b82b9f852b 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -1,5 +1,6 @@ import { LiveObject, LiveObjectData } from './liveobject'; import { LiveObjects } from './liveobjects'; +import { StateCounter, StateCounterOp, StateOperation, StateOperationAction } from './statemessage'; export interface LiveCounterData extends LiveObjectData { data: number; @@ -33,7 +34,79 @@ export class LiveCounter extends LiveObject { this._created = created; } + /** + * @internal + */ + applyOperation(op: StateOperation): void { + if (op.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Cannot apply state operation with objectId=${op.objectId}, to this LiveCounter with objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + + switch (op.action) { + case StateOperationAction.COUNTER_CREATE: + this._applyCounterCreate(op.counter); + break; + + case StateOperationAction.COUNTER_INC: + if (this._client.Utils.isNil(op.counterOp)) { + this._throwNoPayloadError(op); + } else { + this._applyCounterInc(op.counterOp); + } + break; + + default: + throw new this._client.ErrorInfo( + `Invalid ${op.action} op for LiveCounter objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + } + protected _getZeroValueData(): LiveCounterData { return { data: 0 }; } + + private _throwNoPayloadError(op: StateOperation): void { + throw new this._client.ErrorInfo( + `No payload found for ${op.action} op for LiveCounter objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + + private _applyCounterCreate(op: StateCounter | undefined): void { + if (this.isCreated()) { + // skip COUNTER_CREATE op if this counter is already created + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveCounter._applyCounterCreate()', + `skipping applying COUNTER_CREATE op on a counter instance as it is already created; objectId=${this._objectId}`, + ); + return; + } + + if (this._client.Utils.isNil(op)) { + // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly 0 in this case. + // we need to SUM the initial value to the current value due to the reasons below, but since it's a 0, we can skip addition operation + this.setCreated(true); + return; + } + + // note that it is intentional to SUM the incoming count from the create op. + // if we get here, it means that current counter instance wasn't initialized from the COUNTER_CREATE op, + // so it is missing the initial value that we're going to add now. + this._dataRef.data += op.count ?? 0; + this.setCreated(true); + } + + private _applyCounterInc(op: StateCounterOp): void { + this._dataRef.data += op.amount; + } } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 8c13b6d403..30c943c3a4 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -1,7 +1,15 @@ import { LiveObject, LiveObjectData } from './liveobject'; import { LiveObjects } from './liveobjects'; -import { MapSemantics, StateValue } from './statemessage'; -import { Timeserial } from './timeserial'; +import { + MapSemantics, + StateMap, + StateMapOp, + StateMessage, + StateOperation, + StateOperationAction, + StateValue, +} from './statemessage'; +import { DefaultTimeserial, Timeserial } from './timeserial'; export interface ObjectIdStateData { /** A reference to another state object, used to support composable state objects. */ @@ -82,7 +90,165 @@ export class LiveMap extends LiveObject { return size; } + /** + * @internal + */ + applyOperation(op: StateOperation, msg: StateMessage): void { + if (op.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Cannot apply state operation with objectId=${op.objectId}, to this LiveMap with objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + + switch (op.action) { + case StateOperationAction.MAP_CREATE: + this._applyMapCreate(op.map); + break; + + case StateOperationAction.MAP_SET: + if (this._client.Utils.isNil(op.mapOp)) { + this._throwNoPayloadError(op); + } else { + this._applyMapSet(op.mapOp, msg.serial); + } + break; + + case StateOperationAction.MAP_REMOVE: + if (this._client.Utils.isNil(op.mapOp)) { + this._throwNoPayloadError(op); + } else { + this._applyMapRemove(op.mapOp, msg.serial); + } + break; + + default: + throw new this._client.ErrorInfo( + `Invalid ${op.action} op for LiveMap objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + } + protected _getZeroValueData(): LiveMapData { return { data: new Map() }; } + + private _throwNoPayloadError(op: StateOperation): void { + throw new this._client.ErrorInfo( + `No payload found for ${op.action} op for LiveMap objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + + private _applyMapCreate(op: StateMap | undefined): void { + if (this._client.Utils.isNil(op)) { + // if a map object is missing for the MAP_CREATE op, the initial value is implicitly an empty map. + // in this case there is nothing to merge into the current map, so we can just end processing the op. + return; + } + + if (this._semantics !== op.semantics) { + throw new this._client.ErrorInfo( + `Cannot apply MAP_CREATE op on LiveMap objectId=${this.getObjectId()}; map's semantics=${this._semantics}, but op expected ${op.semantics}`, + 50000, + 500, + ); + } + + // in order to apply MAP_CREATE op for an existing map, we should merge their underlying entries keys. + // we can do this by iterating over entries from MAP_CREATE op and apply changes on per-key basis as if we had MAP_SET, MAP_REMOVE operations. + Object.entries(op.entries ?? {}).forEach(([key, entry]) => { + // for MAP_CREATE op we must use dedicated timeserial field available on an entry, instead of a timeserial on a message + const opOriginTimeserial = entry.timeserial; + if (entry.tombstone === true) { + // entry in MAP_CREATE op is deleted, try to apply MAP_REMOVE op + this._applyMapRemove({ key }, opOriginTimeserial); + } else { + // entry in MAP_CREATE op is not deleted, try to set it via MAP_SET op + this._applyMapSet({ key, data: entry.data }, opOriginTimeserial); + } + }); + } + + private _applyMapSet(op: StateMapOp, opOriginTimeserialStr: string | undefined): void { + const { ErrorInfo, Utils } = this._client; + + const opTimeserial = DefaultTimeserial.calculateTimeserial(this._client, opOriginTimeserialStr); + const existingEntry = this._dataRef.data.get(op.key); + if (existingEntry && opTimeserial.before(existingEntry.timeserial)) { + // the operation's origin timeserial < the entry's timeserial, ignore the operation. + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveMap._applyMapSet()', + `skipping updating key="${op.key}" as existing key entry has greater timeserial: ${existingEntry.timeserial.toString()}, than the op: ${opOriginTimeserialStr}; objectId=${this._objectId}`, + ); + return; + } + + if (Utils.isNil(op.data) || (Utils.isNil(op.data.value) && Utils.isNil(op.data.objectId))) { + throw new ErrorInfo( + `Invalid state data for MAP_SET op on objectId=${this.getObjectId()} on key=${op.key}`, + 50000, + 500, + ); + } + + let liveData: StateData; + if (!Utils.isNil(op.data.objectId)) { + liveData = { objectId: op.data.objectId } as ObjectIdStateData; + // this MAP_SET op is setting a key to point to another object via its object id, + // but it is possible that we don't have the corresponding object in the pool yet (for example, we haven't seen the *_CREATE op for it). + // we don't want to return undefined from this map's .get() method even if we don't have the object, + // so instead we create a zero-value object for that object id if it not exists. + this._liveObjects.getPool().createZeroValueObjectIfNotExists(op.data.objectId); + } else { + liveData = { encoding: op.data.encoding, value: op.data.value } as ValueStateData; + } + + if (existingEntry) { + existingEntry.tombstone = false; + existingEntry.timeserial = opTimeserial; + existingEntry.data = liveData; + } else { + const newEntry: MapEntry = { + tombstone: false, + timeserial: opTimeserial, + data: liveData, + }; + this._dataRef.data.set(op.key, newEntry); + } + } + + private _applyMapRemove(op: StateMapOp, opOriginTimeserialStr: string | undefined): void { + const opTimeserial = DefaultTimeserial.calculateTimeserial(this._client, opOriginTimeserialStr); + const existingEntry = this._dataRef.data.get(op.key); + if (existingEntry && opTimeserial.before(existingEntry.timeserial)) { + // the operation's origin timeserial < the entry's timeserial, ignore the operation. + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveMap._applyMapRemove()', + `skipping removing key="${op.key}" as existing key entry has greater timeserial: ${existingEntry.timeserial.toString()}, than the op: ${opOriginTimeserialStr}; objectId=${this._objectId}`, + ); + return; + } + + if (existingEntry) { + existingEntry.tombstone = true; + existingEntry.timeserial = opTimeserial; + existingEntry.data = undefined; + } else { + const newEntry: MapEntry = { + tombstone: true, + timeserial: opTimeserial, + data: undefined, + }; + this._dataRef.data.set(op.key, newEntry); + } + } } diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index d062fec37b..2e33cb0b23 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -1,10 +1,13 @@ +import type BaseClient from 'common/lib/client/baseclient'; import { LiveObjects } from './liveobjects'; +import { StateMessage, StateOperation } from './statemessage'; export interface LiveObjectData { data: any; } export abstract class LiveObject { + protected _client: BaseClient; protected _dataRef: T; protected _objectId: string; protected _regionalTimeserial?: string; @@ -14,6 +17,7 @@ export abstract class LiveObject { initialData?: T | null, objectId?: string, ) { + this._client = this._liveObjects.getClient(); this._dataRef = initialData ?? this._getZeroValueData(); this._objectId = objectId ?? this._createObjectId(); } @@ -51,5 +55,9 @@ export abstract class LiveObject { return Math.random().toString().substring(2); } + /** + * @internal + */ + abstract applyOperation(op: StateOperation, msg: StateMessage): void; protected abstract _getZeroValueData(): T; } diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index f4345d61e9..959411a73f 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -1,7 +1,9 @@ import type BaseClient from 'common/lib/client/baseclient'; +import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; import { LiveObjects } from './liveobjects'; +import { ObjectId } from './objectid'; import { MapSemantics } from './statemessage'; export const ROOT_OBJECT_ID = 'root'; @@ -40,6 +42,27 @@ export class LiveObjectsPool { this._pool = this._getInitialPool(); } + createZeroValueObjectIfNotExists(objectId: string): void { + if (this.get(objectId)) { + return; + } + + const parsedObjectId = ObjectId.fromString(this._client, objectId); + let zeroValueObject: LiveObject; + switch (parsedObjectId.type) { + case 'map': { + zeroValueObject = new LiveMap(this._liveObjects, MapSemantics.LWW, null, objectId); + break; + } + + case 'counter': + zeroValueObject = new LiveCounter(this._liveObjects, false, null, objectId); + break; + } + + this.set(objectId, zeroValueObject); + } + private _getInitialPool(): Map { const pool = new Map(); const root = new LiveMap(this._liveObjects, MapSemantics.LWW, null, ROOT_OBJECT_ID); From 9f98ef68ae043dcadde4e2d84b4d30bdbf9c02ee Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 21 Oct 2024 13:20:17 +0100 Subject: [PATCH 042/166] Preparation for applying incoming state operations - add static LiveMap.liveMapDataFromMapEntries() method with logic previously used in SyncLiveObjectsDataPool. This method will be used in other places to create LiveMapData - minor refactoring and log improvements --- src/common/lib/client/realtimechannel.ts | 2 +- src/plugins/liveobjects/livemap.ts | 30 ++++++++++++++ src/plugins/liveobjects/liveobjects.ts | 8 ++-- src/plugins/liveobjects/statemessage.ts | 2 +- .../liveobjects/syncliveobjectsdatapool.ts | 41 ++++--------------- 5 files changed, 45 insertions(+), 38 deletions(-) diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index 928650ad6b..d9cbabd812 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -649,7 +649,7 @@ class RealtimeChannel extends EventEmitter { } } - this._liveObjects.handleStateSyncMessage(stateMessages, message.channelSerial); + this._liveObjects.handleStateSyncMessages(stateMessages, message.channelSerial); break; } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 30c943c3a4..90582d3a7c 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -1,8 +1,10 @@ +import type BaseClient from 'common/lib/client/baseclient'; import { LiveObject, LiveObjectData } from './liveobject'; import { LiveObjects } from './liveobjects'; import { MapSemantics, StateMap, + StateMapEntry, StateMapOp, StateMessage, StateOperation, @@ -48,6 +50,34 @@ export class LiveMap extends LiveObject { super(liveObjects, initialData, objectId); } + static liveMapDataFromMapEntries(client: BaseClient, entries: Record): LiveMapData { + const liveMapData: LiveMapData = { + data: new Map(), + }; + + // need to iterate over entries manually to work around optional parameters from state object entries type + Object.entries(entries ?? {}).forEach(([key, entry]) => { + let liveData: StateData; + if (typeof entry.data.objectId !== 'undefined') { + liveData = { objectId: entry.data.objectId } as ObjectIdStateData; + } else { + liveData = { encoding: entry.data.encoding, value: entry.data.value } as ValueStateData; + } + + const liveDataEntry: MapEntry = { + ...entry, + timeserial: DefaultTimeserial.calculateTimeserial(client, entry.timeserial), + // true only if we received explicit true. otherwise always false + tombstone: entry.tombstone === true, + data: liveData, + }; + + liveMapData.data.set(key, liveDataEntry); + }); + + return liveMapData; + } + /** * Returns the value associated with the specified key in the underlying Map object. * If no element is associated with the specified key, undefined is returned. diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 11a5ef3a69..a8e8ab885a 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -67,13 +67,13 @@ export class LiveObjects { /** * @internal */ - handleStateSyncMessage(stateMessages: StateMessage[], syncChannelSerial: string | null | undefined): void { + handleStateSyncMessages(stateMessages: StateMessage[], syncChannelSerial: string | null | undefined): void { const { syncId, syncCursor } = this._parseSyncChannelSerial(syncChannelSerial); if (this._currentSyncId !== syncId) { this._startNewSync(syncId, syncCursor); } - this._syncLiveObjectsDataPool.applyStateMessages(stateMessages); + this._syncLiveObjectsDataPool.applyStateSyncMessages(stateMessages); // if this is the last (or only) message in a sequence of sync updates, end the sync if (!syncCursor) { @@ -93,7 +93,7 @@ export class LiveObjects { ); if (hasState) { - this._startNewSync(undefined); + this._startNewSync(); } else { // no HAS_STATE flag received on attach, can end SYNC sequence immediately // and treat it as no state on a channel @@ -190,7 +190,7 @@ export class LiveObjects { break; default: - throw new this._client.ErrorInfo(`Unknown live object type: ${objectType}`, 40000, 400); + throw new this._client.ErrorInfo(`Unknown live object type: ${objectType}`, 50000, 500); } newObject.setRegionalTimeserial(entry.regionalTimeserial); diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 858f87ea1c..2fd293a336 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -127,7 +127,7 @@ export class StateMessage { operation?: StateOperation; /** Describes the instantaneous state of an object. */ object?: StateObject; - /** Timeserial format */ + /** Timeserial format. Contains the origin timeserial for this state message. */ serial?: string; constructor(private _platform: typeof Platform) {} diff --git a/src/plugins/liveobjects/syncliveobjectsdatapool.ts b/src/plugins/liveobjects/syncliveobjectsdatapool.ts index 22e83c7a19..8f0f8d6485 100644 --- a/src/plugins/liveobjects/syncliveobjectsdatapool.ts +++ b/src/plugins/liveobjects/syncliveobjectsdatapool.ts @@ -1,11 +1,10 @@ import type BaseClient from 'common/lib/client/baseclient'; -import RealtimeChannel from 'common/lib/client/realtimechannel'; +import type RealtimeChannel from 'common/lib/client/realtimechannel'; import { LiveCounterData } from './livecounter'; -import { LiveMapData, MapEntry, ObjectIdStateData, StateData, ValueStateData } from './livemap'; +import { LiveMap } from './livemap'; import { LiveObjectData } from './liveobject'; import { LiveObjects } from './liveobjects'; import { MapSemantics, StateMessage, StateObject } from './statemessage'; -import { DefaultTimeserial } from './timeserial'; export interface LiveObjectDataEntry { objectData: LiveObjectData; @@ -55,14 +54,14 @@ export class SyncLiveObjectsDataPool { this._pool.clear(); } - applyStateMessages(stateMessages: StateMessage[]): void { + applyStateSyncMessages(stateMessages: StateMessage[]): void { for (const stateMessage of stateMessages) { if (!stateMessage.object) { this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MAJOR, - 'LiveObjects.SyncLiveObjectsDataPool.applyStateMessages()', - `state message is received during SYNC without 'object' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + 'LiveObjects.SyncLiveObjectsDataPool.applyStateSyncMessages()', + `state object message is received during SYNC without 'object' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, ); continue; } @@ -76,9 +75,9 @@ export class SyncLiveObjectsDataPool { } else { this._client.Logger.logAction( this._client.logger, - this._client.Logger.LOG_MINOR, - 'LiveObjects.SyncLiveObjectsDataPool.applyStateMessages()', - `received unsupported state object message during SYNC, expected 'counter' or 'map' to be present; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + this._client.Logger.LOG_MAJOR, + 'LiveObjects.SyncLiveObjectsDataPool.applyStateSyncMessages()', + `received unsupported state object message during SYNC, expected 'counter' or 'map' to be present, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, ); } } @@ -102,29 +101,7 @@ export class SyncLiveObjectsDataPool { private _createLiveMapDataEntry(stateObject: StateObject): LiveMapDataEntry { const map = stateObject.map!; - - const objectData: LiveMapData = { - data: new Map(), - }; - // need to iterate over entries manually to work around optional parameters from state object entries type - Object.entries(map.entries ?? {}).forEach(([key, entryFromMessage]) => { - let liveData: StateData; - if (typeof entryFromMessage.data.objectId !== 'undefined') { - liveData = { objectId: entryFromMessage.data.objectId } as ObjectIdStateData; - } else { - liveData = { encoding: entryFromMessage.data.encoding, value: entryFromMessage.data.value } as ValueStateData; - } - - const liveDataEntry: MapEntry = { - ...entryFromMessage, - timeserial: DefaultTimeserial.calculateTimeserial(this._client, entryFromMessage.timeserial), - // true only if we received explicit true. otherwise always false - tombstone: entryFromMessage.tombstone === true, - data: liveData, - }; - - objectData.data.set(key, liveDataEntry); - }); + const objectData = LiveMap.liveMapDataFromMapEntries(this._client, map.entries ?? {}); const newEntry: LiveMapDataEntry = { objectData, From 0b646e2e8cdcad7da7c8544b6f8505a3cb38b98a Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 21 Oct 2024 13:17:54 +0100 Subject: [PATCH 043/166] Implement application of incoming state operation messages outside of sync sequence Resolves DTP-956 --- src/common/lib/client/realtimechannel.ts | 33 ++++++++ src/plugins/liveobjects/liveobjects.ts | 15 +++- src/plugins/liveobjects/liveobjectspool.ts | 97 +++++++++++++++++++++- 3 files changed, 143 insertions(+), 2 deletions(-) diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index d9cbabd812..4cdcd23126 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -621,6 +621,39 @@ class RealtimeChannel extends EventEmitter { break; } + case actions.STATE: { + if (!this._liveObjects) { + return; + } + + const { id, connectionId, timestamp } = message; + const options = this.channelOptions; + + const stateMessages = message.state ?? []; + for (let i = 0; i < stateMessages.length; i++) { + try { + const stateMessage = stateMessages[i]; + + await this.client._LiveObjectsPlugin?.StateMessage.decode(stateMessage, options, decodeData); + + if (!stateMessage.connectionId) stateMessage.connectionId = connectionId; + if (!stateMessage.timestamp) stateMessage.timestamp = timestamp; + if (!stateMessage.id) stateMessage.id = id + ':' + i; + } catch (e) { + Logger.logAction( + this.logger, + Logger.LOG_ERROR, + 'RealtimeChannel.processMessage()', + (e as Error).toString(), + ); + } + } + + this._liveObjects.handleStateMessages(stateMessages); + + break; + } + case actions.STATE_SYNC: { if (!this._liveObjects) { return; diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index a8e8ab885a..0fb0886ed1 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -1,7 +1,7 @@ import type BaseClient from 'common/lib/client/baseclient'; import type RealtimeChannel from 'common/lib/client/realtimechannel'; -import type * as API from '../../../ably'; import type EventEmitter from 'common/lib/util/eventemitter'; +import type * as API from '../../../ably'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; @@ -81,6 +81,17 @@ export class LiveObjects { } } + /** + * @internal + */ + handleStateMessages(stateMessages: StateMessage[]): void { + if (this._syncInProgress) { + // TODO: handle buffering of state messages during SYNC + } + + this._liveObjectsPool.applyStateMessages(stateMessages); + } + /** * @internal */ @@ -131,6 +142,8 @@ export class LiveObjects { } private _endSync(): void { + // TODO: handle applying buffered state messages when SYNC is finished + this._applySync(); this._syncLiveObjectsDataPool.reset(); this._currentSyncId = undefined; diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index 959411a73f..02123f17c8 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -1,10 +1,11 @@ import type BaseClient from 'common/lib/client/baseclient'; +import type RealtimeChannel from 'common/lib/client/realtimechannel'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; import { LiveObjects } from './liveobjects'; import { ObjectId } from './objectid'; -import { MapSemantics } from './statemessage'; +import { MapSemantics, StateMessage, StateOperation, StateOperationAction } from './statemessage'; export const ROOT_OBJECT_ID = 'root'; @@ -13,10 +14,12 @@ export const ROOT_OBJECT_ID = 'root'; */ export class LiveObjectsPool { private _client: BaseClient; + private _channel: RealtimeChannel; private _pool: Map; constructor(private _liveObjects: LiveObjects) { this._client = this._liveObjects.getClient(); + this._channel = this._liveObjects.getChannel(); this._pool = this._getInitialPool(); } @@ -63,10 +66,102 @@ export class LiveObjectsPool { this.set(objectId, zeroValueObject); } + applyStateMessages(stateMessages: StateMessage[]): void { + for (const stateMessage of stateMessages) { + if (!stateMessage.operation) { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MAJOR, + 'LiveObjects.LiveObjectsPool.applyStateMessages()', + `state operation message is received without 'operation' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + ); + continue; + } + + const stateOperation = stateMessage.operation; + + switch (stateOperation.action) { + case StateOperationAction.MAP_CREATE: + case StateOperationAction.COUNTER_CREATE: + if (this.get(stateOperation.objectId)) { + // object wich such id already exists (we may have created a zero-value object before, or this is a duplicate *_CREATE op), + // so delegate application of the op to that object + // TODO: invoke subscription callbacks for an object when applied + this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); + break; + } + + // otherwise we can create new objects in the pool + if (stateOperation.action === StateOperationAction.MAP_CREATE) { + this._handleMapCreate(stateOperation); + } + + if (stateOperation.action === StateOperationAction.COUNTER_CREATE) { + this._handleCounterCreate(stateOperation); + } + break; + + case StateOperationAction.MAP_SET: + case StateOperationAction.MAP_REMOVE: + case StateOperationAction.COUNTER_INC: + // we can receive an op for an object id we don't have yet in the pool. instead of buffering such operations, + // we create a zero-value object for the provided object id, and apply operation for that zero-value object. + // when we eventually receive a corresponding *_CREATE op for that object, its application will be handled by that zero-value object. + this.createZeroValueObjectIfNotExists(stateOperation.objectId); + // TODO: invoke subscription callbacks for an object when applied + this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); + break; + + default: + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MAJOR, + 'LiveObjects.LiveObjectsPool.applyStateMessages()', + `received unsupported action in state operation message: ${stateOperation.action}, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + ); + } + } + } + private _getInitialPool(): Map { const pool = new Map(); const root = new LiveMap(this._liveObjects, MapSemantics.LWW, null, ROOT_OBJECT_ID); pool.set(root.getObjectId(), root); return pool; } + + private _handleCounterCreate(stateOperation: StateOperation): void { + let counter: LiveCounter; + if (this._client.Utils.isNil(stateOperation.counter)) { + // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly 0 in this case. + counter = new LiveCounter(this._liveObjects, true, { data: 0 }, stateOperation.objectId); + } else { + counter = new LiveCounter( + this._liveObjects, + true, + { data: stateOperation.counter.count ?? 0 }, + stateOperation.objectId, + ); + } + + this.set(stateOperation.objectId, counter); + } + + private _handleMapCreate(stateOperation: StateOperation): void { + let map: LiveMap; + if (this._client.Utils.isNil(stateOperation.map)) { + // if a map object is missing for the MAP_CREATE op, the initial value is implicitly an empty map. + map = new LiveMap(this._liveObjects, MapSemantics.LWW, null, stateOperation.objectId); + } else { + const objectData = LiveMap.liveMapDataFromMapEntries(this._client, stateOperation.map.entries ?? {}); + map = new LiveMap( + this._liveObjects, + stateOperation.map.semantics ?? MapSemantics.LWW, + objectData, + stateOperation.objectId, + ); + } + + this.set(stateOperation.objectId, map); + } } From ef54890d5445d26cbee319c797416f0a1c1589e1 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 21 Oct 2024 15:50:28 +0100 Subject: [PATCH 044/166] Preparation for LiveObjects tests for applying incoming operations - LiveObjectsHelper refactoring - timeserials format fix in existing LiveObjects tests (add seriesId part) --- test/common/modules/live_objects_helper.js | 92 ++++++++++++++-------- test/realtime/live_objects.test.js | 16 ++-- 2 files changed, 71 insertions(+), 37 deletions(-) diff --git a/test/common/modules/live_objects_helper.js b/test/common/modules/live_objects_helper.js index 2f6ffc3843..b2793804e6 100644 --- a/test/common/modules/live_objects_helper.js +++ b/test/common/modules/live_objects_helper.js @@ -17,6 +17,10 @@ define(['shared_helper'], function (Helper) { } class LiveObjectsHelper { + constructor(helper) { + this._rest = helper.AblyRest({ useBinaryProtocol: false }); + } + /** * Creates next LiveObjects state tree on a provided channel name: * @@ -27,39 +31,37 @@ define(['shared_helper'], function (Helper) { * root "initialValueCounter" -> Counter#2 count=10 * root "referencedCounter" -> Counter#3 count=20 */ - async initForChannel(helper, channelName) { - const rest = helper.AblyRest({ useBinaryProtocol: false }); - - const emptyCounter = await this._createAndSetOnMap(rest, channelName, { + async initForChannel(channelName) { + const emptyCounter = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'emptyCounter', - createOp: this._counterCreateOp(), + createOp: this.counterCreateOp(), }); - const initialValueCounter = await this._createAndSetOnMap(rest, channelName, { + const initialValueCounter = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'initialValueCounter', - createOp: this._counterCreateOp({ count: 10 }), + createOp: this.counterCreateOp({ count: 10 }), }); - const referencedCounter = await this._createAndSetOnMap(rest, channelName, { + const referencedCounter = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'referencedCounter', - createOp: this._counterCreateOp({ count: 20 }), + createOp: this.counterCreateOp({ count: 20 }), }); - const emptyMap = await this._createAndSetOnMap(rest, channelName, { + const emptyMap = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'emptyMap', - createOp: this._mapCreateOp(), + createOp: this.mapCreateOp(), }); - const referencedMap = await this._createAndSetOnMap(rest, channelName, { + const referencedMap = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'referencedMap', - createOp: this._mapCreateOp({ entries: { counterKey: { data: { objectId: referencedCounter.objectId } } } }), + createOp: this.mapCreateOp({ entries: { counterKey: { data: { objectId: referencedCounter.objectId } } } }), }); - const valuesMap = await this._createAndSetOnMap(rest, channelName, { + const valuesMap = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'valuesMap', - createOp: this._mapCreateOp({ + createOp: this.mapCreateOp({ entries: { stringKey: { data: { value: 'stringValue' } }, emptyStringKey: { data: { value: '' } }, @@ -77,20 +79,19 @@ define(['shared_helper'], function (Helper) { }); } - async _createAndSetOnMap(rest, channelName, opts) { + async createAndSetOnMap(channelName, opts) { const { mapObjectId, key, createOp } = opts; - const createResult = await this._stateRequest(rest, channelName, createOp); - await this._stateRequest( - rest, + const createResult = await this.stateRequest(channelName, createOp); + await this.stateRequest( channelName, - this._mapSetOp({ objectId: mapObjectId, key, data: { objectId: createResult.objectId } }), + this.mapSetOp({ objectId: mapObjectId, key, data: { objectId: createResult.objectId } }), ); return createResult; } - _mapCreateOp(opts) { + mapCreateOp(opts) { const { objectId, entries } = opts ?? {}; const op = { operation: { @@ -107,26 +108,38 @@ define(['shared_helper'], function (Helper) { return op; } - _mapSetOp(opts) { + mapSetOp(opts) { const { objectId, key, data } = opts ?? {}; const op = { operation: { action: ACTIONS.MAP_SET, objectId, + mapOp: { + key, + data, + }, }, }; - if (key && data) { - op.operation.mapOp = { - key, - data, - }; - } + return op; + } + + mapRemoveOp(opts) { + const { objectId, key } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.MAP_REMOVE, + objectId, + mapOp: { + key, + }, + }, + }; return op; } - _counterCreateOp(opts) { + counterCreateOp(opts) { const { objectId, count } = opts ?? {}; const op = { operation: { @@ -143,7 +156,22 @@ define(['shared_helper'], function (Helper) { return op; } - async _stateRequest(rest, channelName, opBody) { + counterIncOp(opts) { + const { objectId, amount } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.COUNTER_INC, + objectId, + counterOp: { + amount, + }, + }, + }; + + return op; + } + + async stateRequest(channelName, opBody) { if (Array.isArray(opBody)) { throw new Error(`Only single object state requests are supported`); } @@ -151,7 +179,7 @@ define(['shared_helper'], function (Helper) { const method = 'post'; const path = `/channels/${channelName}/state`; - const response = await rest.request(method, path, 3, null, opBody, null); + const response = await this._rest.request(method, path, 3, null, opBody, null); if (response.success) { // only one operation in request, so need only first item. @@ -167,5 +195,5 @@ define(['shared_helper'], function (Helper) { } } - return (module.exports = new LiveObjectsHelper()); + return (module.exports = LiveObjectsHelper); }); diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index e3c0d078ea..ae2515738c 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -36,7 +36,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], return; } - LiveObjectsHelper.initForChannel(helper, liveObjectsFixturesChannel) + new LiveObjectsHelper(helper) + .initForChannel(liveObjectsFixturesChannel) .then(done) .catch((err) => done(err)); }); @@ -77,7 +78,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { object: { objectId: 'root', - regionalTimeserial: '@0-0', + regionalTimeserial: 'a@0-0', map: {}, }, }, @@ -220,7 +221,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // wait for initial STATE_SYNC sequence to complete await liveObjects.getRoot(); - // inject STATE_SYNC message to emulate start of new sequence + // inject STATE_SYNC message to emulate start of a new sequence helper.recordPrivateApi('call.channel.processMessage'); helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); await channel.processMessage( @@ -259,11 +260,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { object: { objectId: 'root', - regionalTimeserial: '@0-0', + regionalTimeserial: 'a@0-0', map: { entries: { key: { - timeserial: '@0-0', + timeserial: 'a@0-0', data: { value: 1, }, @@ -285,6 +286,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, client); }); + /** @nospec */ it('builds state object tree from STATE_SYNC sequence on channel attachment', async function () { const helper = this.test.helper; const client = RealtimeWithLiveObjects(helper); @@ -338,6 +340,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, client); }); + /** @nospec */ it('LiveCounter is initialized with initial value from STATE_SYNC sequence', async function () { const helper = this.test.helper; const client = RealtimeWithLiveObjects(helper); @@ -362,6 +365,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, client); }); + /** @nospec */ it('LiveMap is initialized with initial value from STATE_SYNC sequence', async function () { const helper = this.test.helper; const client = RealtimeWithLiveObjects(helper); @@ -408,6 +412,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, client); }); + /** @nospec */ it('LiveMaps can reference the same object in their keys', async function () { const helper = this.test.helper; const client = RealtimeWithLiveObjects(helper); @@ -447,6 +452,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); }); + /** @nospec */ it('can attach to channel with LiveObjects state modes', async function () { const helper = this.test.helper; const client = helper.AblyRealtime(); From bda102caab6e67219d8a38374f6b773ab8dd0362 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 22 Oct 2024 06:34:25 +0100 Subject: [PATCH 045/166] Add LiveObjects tests for applying incoming operation messages outside sync sequence --- test/realtime/live_objects.test.js | 499 ++++++++++++++++++++++++++++- 1 file changed, 498 insertions(+), 1 deletion(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index ae2515738c..c0da311f30 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -53,7 +53,51 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); /** @nospec */ - it('doesn’t break when it receives a STATE_SYNC ProtocolMessage', async function () { + it(`doesn't break when it receives a STATE ProtocolMessage`, async function () { + const helper = this.test.helper; + const testClient = helper.AblyRealtime(); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const testChannel = testClient.channels.get('channel'); + await testChannel.attach(); + + const receivedMessagePromise = new Promise((resolve) => testChannel.subscribe(resolve)); + + const publishClient = helper.AblyRealtime(); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + // inject STATE message that should be ignored and not break anything without LiveObjects plugin + helper.recordPrivateApi('call.channel.processMessage'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + await testChannel.processMessage( + createPM({ + action: 19, + channel: 'channel', + channelSerial: 'serial:', + state: [ + { + operation: { + action: 1, + objectId: 'root', + mapOp: { key: 'stringKey', data: { value: 'stringValue' } }, + }, + serial: 'a@0-0', + }, + ], + }), + ); + + const publishChannel = publishClient.channels.get('channel'); + await publishChannel.publish(null, 'test'); + + // regular message subscriptions should still work after processing STATE_SYNC message without LiveObjects plugin + await receivedMessagePromise; + }, publishClient); + }, testClient); + }); + + /** @nospec */ + it(`doesn't break when it receives a STATE_SYNC ProtocolMessage`, async function () { const helper = this.test.helper; const testClient = helper.AblyRealtime(); @@ -450,6 +494,459 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ); }, client); }); + + const primitiveKeyData = [ + { key: 'stringKey', data: { value: 'stringValue' } }, + { key: 'emptyStringKey', data: { value: '' } }, + { + key: 'bytesKey', + data: { value: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9', encoding: 'base64' }, + }, + { key: 'emptyBytesKey', data: { value: '', encoding: 'base64' } }, + { key: 'numberKey', data: { value: 1 } }, + { key: 'zeroKey', data: { value: 0 } }, + { key: 'trueKey', data: { value: true } }, + { key: 'falseKey', data: { value: false } }, + ]; + const primitiveMapsFixtures = [ + { name: 'emptyMap' }, + { + name: 'valuesMap', + entries: primitiveKeyData.reduce((acc, v) => { + acc[v.key] = { data: v.data }; + return acc; + }, {}), + }, + ]; + const countersFixtures = [ + { name: 'emptyCounter' }, + { name: 'zeroCounter', count: 0 }, + { name: 'valueCounter', count: 10 }, + { name: 'negativeValueCounter', count: -10 }, + { name: 'maxSafeIntegerCounter', count: Number.MAX_SAFE_INTEGER }, + { name: 'negativeMaxSafeIntegerCounter', count: -Number.MAX_SAFE_INTEGER }, + ]; + const applyOperationsScenarios = [ + { + description: 'MAP_CREATE with primitives', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + // LiveObjects public API allows us to check value of objects we've created based on MAP_CREATE ops + // if we assign those objects to another map (root for example), as there is no way to access those objects from the internal pool directly. + // however, in this test we put heavy focus on the data that is being created as the result of the MAP_CREATE op. + + // check no maps exist on root + primitiveMapsFixtures.forEach((fixture) => { + const key = fixture.name; + expect(root.get(key, `Check "${key}" key doesn't exist on root before applying MAP_CREATE ops`)).to.not + .exist; + }); + + // create new maps and set on root + await Promise.all( + primitiveMapsFixtures.map((fixture) => + liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: fixture.name, + createOp: liveObjectsHelper.mapCreateOp({ entries: fixture.entries }), + }), + ), + ); + + // check created maps + primitiveMapsFixtures.forEach((fixture) => { + const key = fixture.name; + const mapObj = root.get(key); + + // check all maps exist on root + expect(mapObj, `Check map at "${key}" key in root exists`).to.exist; + expect(mapObj.constructor.name).to.equal( + 'LiveMap', + `Check map at "${key}" key in root is of type LiveMap`, + ); + + // check primitive maps have correct values + expect(mapObj.size()).to.equal( + Object.keys(fixture.entries ?? {}).length, + `Check map "${key}" has correct number of keys`, + ); + + Object.entries(fixture.entries ?? {}).forEach(([key, keyData]) => { + if (keyData.data.encoding) { + expect( + BufferUtils.areBuffersEqual(mapObj.get(key), BufferUtils.base64Decode(keyData.data.value)), + `Check map "${key}" has correct value for "${key}" key`, + ).to.be.true; + } else { + expect(mapObj.get(key)).to.equal( + keyData.data.value, + `Check map "${key}" has correct value for "${key}" key`, + ); + } + }); + }); + }, + }, + + { + description: 'MAP_CREATE with object ids', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + const withReferencesMapKey = 'withReferencesMap'; + + // LiveObjects public API allows us to check value of objects we've created based on MAP_CREATE ops + // if we assign those objects to another map (root for example), as there is no way to access those objects from the internal pool directly. + // however, in this test we put heavy focus on the data that is being created as the result of the MAP_CREATE op. + + // check map does not exist on root + expect( + root.get( + withReferencesMapKey, + `Check "${withReferencesMapKey}" key doesn't exist on root before applying MAP_CREATE ops`, + ), + ).to.not.exist; + + // create map with references. need to created referenced objects first to obtain their object ids + const { objectId: referencedMapObjectId } = await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapCreateOp({ entries: { stringKey: { data: { value: 'stringValue' } } } }), + ); + const { objectId: referencedCounterObjectId } = await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.counterCreateOp({ count: 1 }), + ); + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: withReferencesMapKey, + createOp: liveObjectsHelper.mapCreateOp({ + entries: { + mapReference: { data: { objectId: referencedMapObjectId } }, + counterReference: { data: { objectId: referencedCounterObjectId } }, + }, + }), + }); + + // check map with references exist on root + const withReferencesMap = root.get(withReferencesMapKey); + expect(withReferencesMap, `Check map at "${withReferencesMapKey}" key in root exists`).to.exist; + expect(withReferencesMap.constructor.name).to.equal( + 'LiveMap', + `Check map at "${withReferencesMapKey}" key in root is of type LiveMap`, + ); + + // check map with references has correct values + expect(withReferencesMap.size()).to.equal( + 2, + `Check map "${withReferencesMapKey}" has correct number of keys`, + ); + + const referencedCounter = withReferencesMap.get('counterReference'); + const referencedMap = withReferencesMap.get('mapReference'); + + expect(referencedCounter, `Check counter at "counterReference" exists`).to.exist; + expect(referencedCounter.constructor.name).to.equal( + 'LiveCounter', + `Check counter at "counterReference" key is of type LiveCounter`, + ); + expect(referencedCounter.value()).to.equal(1, 'Check counter at "counterReference" key has correct value'); + + expect(referencedMap, `Check map at "mapReference" key exists`).to.exist; + expect(referencedMap.constructor.name).to.equal( + 'LiveMap', + `Check map at "mapReference" key is of type LiveMap`, + ); + expect(referencedMap.size()).to.equal(1, 'Check map at "mapReference" key has correct number of keys'); + expect(referencedMap.get('stringKey')).to.equal( + 'stringValue', + 'Check map at "mapReference" key has correct "stringKey" value', + ); + }, + }, + + { + description: 'MAP_SET with primitives', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + // check root is empty before ops + primitiveKeyData.forEach((keyData) => { + expect( + root.get(keyData.key, `Check "${keyData.key}" key doesn't exist on root before applying MAP_SET ops`), + ).to.not.exist; + }); + + // apply MAP_SET ops + await Promise.all( + primitiveKeyData.map((keyData) => + liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapSetOp({ + objectId: 'root', + key: keyData.key, + data: keyData.data, + }), + ), + ), + ); + + // check everything is applied correctly + primitiveKeyData.forEach((keyData) => { + if (keyData.data.encoding) { + expect( + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), + `Check root has correct value for "${keyData.key}" key after MAP_SET op`, + ).to.be.true; + } else { + expect(root.get(keyData.key)).to.equal( + keyData.data.value, + `Check root has correct value for "${keyData.key}" key after MAP_SET op`, + ); + } + }); + }, + }, + + { + description: 'MAP_SET with object ids', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + // check no object ids are set on root + expect( + root.get('keyToCounter', `Check "keyToCounter" key doesn't exist on root before applying MAP_SET ops`), + ).to.not.exist; + expect(root.get('keyToMap', `Check "keyToMap" key doesn't exist on root before applying MAP_SET ops`)).to + .not.exist; + + // create new objects and set on root + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'keyToCounter', + createOp: liveObjectsHelper.counterCreateOp({ count: 1 }), + }); + + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'keyToMap', + createOp: liveObjectsHelper.mapCreateOp({ + entries: { + stringKey: { data: { value: 'stringValue' } }, + }, + }), + }); + + // check root has refs to new objects and they are not zero-value + const counter = root.get('keyToCounter'); + const map = root.get('keyToMap'); + + expect(counter, 'Check counter at "keyToCounter" key in root exists').to.exist; + expect(counter.constructor.name).to.equal( + 'LiveCounter', + 'Check counter at "keyToCounter" key in root is of type LiveCounter', + ); + expect(counter.value()).to.equal(1, 'Check counter at "keyToCounter" key in root has correct value'); + + expect(map, 'Check map at "keyToMap" key in root exists').to.exist; + expect(map.constructor.name).to.equal('LiveMap', 'Check map at "keyToMap" key in root is of type LiveMap'); + expect(map.size()).to.equal(1, 'Check map at "keyToMap" key in root has correct number of keys'); + expect(map.get('stringKey')).to.equal( + 'stringValue', + 'Check map at "keyToMap" key in root has correct "stringKey" value', + ); + }, + }, + + { + description: 'MAP_REMOVE', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + const mapKey = 'map'; + + // create new map and set on root + const { objectId: mapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: mapKey, + createOp: liveObjectsHelper.mapCreateOp({ + entries: { + shouldStay: { data: { value: 'foo' } }, + shouldDelete: { data: { value: 'bar' } }, + }, + }), + }); + + const map = root.get(mapKey); + // check map has expected keys before MAP_REMOVE ops + expect(map.size()).to.equal( + 2, + `Check map at "${mapKey}" key in root has correct number of keys before MAP_REMOVE`, + ); + expect(map.get('shouldStay')).to.equal( + 'foo', + `Check map at "${mapKey}" key in root has correct "shouldStay" value before MAP_REMOVE`, + ); + expect(map.get('shouldDelete')).to.equal( + 'bar', + `Check map at "${mapKey}" key in root has correct "shouldDelete" value before MAP_REMOVE`, + ); + + // send MAP_REMOVE op + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapRemoveOp({ + objectId: mapObjectId, + key: 'shouldDelete', + }), + ); + + // check map has correct keys after MAP_REMOVE ops + expect(map.size()).to.equal( + 1, + `Check map at "${mapKey}" key in root has correct number of keys after MAP_REMOVE`, + ); + expect(map.get('shouldStay')).to.equal( + 'foo', + `Check map at "${mapKey}" key in root has correct "shouldStay" value after MAP_REMOVE`, + ); + expect( + map.get('shouldDelete'), + `Check map at "${mapKey}" key in root has no "shouldDelete" key after MAP_REMOVE`, + ).to.not.exist; + }, + }, + + { + description: 'COUNTER_CREATE', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + // LiveObjects public API allows us to check value of objects we've created based on COUNTER_CREATE ops + // if we assign those objects to another map (root for example), as there is no way to access those objects from the internal pool directly. + // however, in this test we put heavy focus on the data that is being created as the result of the COUNTER_CREATE op. + + // check no counters exist on root + countersFixtures.forEach((fixture) => { + const key = fixture.name; + expect(root.get(key, `Check "${key}" key doesn't exist on root before applying COUNTER_CREATE ops`)).to + .not.exist; + }); + + // create new counters and set on root + await Promise.all( + countersFixtures.map((fixture) => + liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: fixture.name, + createOp: liveObjectsHelper.counterCreateOp({ count: fixture.count }), + }), + ), + ); + + // check created counters + countersFixtures.forEach((fixture) => { + const key = fixture.name; + const counterObj = root.get(key); + + // check all counters exist on root + expect(counterObj, `Check counter at "${key}" key in root exists`).to.exist; + expect(counterObj.constructor.name).to.equal( + 'LiveCounter', + `Check counter at "${key}" key in root is of type LiveCounter`, + ); + + // check counters have correct values + expect(counterObj.value()).to.equal( + // if count was not set, should default to 0 + fixture.count ?? 0, + `Check counter at "${key}" key in root has correct value`, + ); + }); + }, + }, + + { + description: 'COUNTER_INC', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + const counterKey = 'counter'; + let expectedCounterValue = 0; + + // create new counter and set on root + const { objectId: counterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: counterKey, + createOp: liveObjectsHelper.counterCreateOp({ count: expectedCounterValue }), + }); + + const counter = root.get(counterKey); + // check counter has expected value before COUNTER_INC + expect(counter.value()).to.equal( + expectedCounterValue, + `Check counter at "${counterKey}" key in root has correct value before COUNTER_INC`, + ); + + const increments = [ + 1, // value=1 + 10, // value=11 + 100, // value=111 + 1000000, // value=1000111 + -1000111, // value=0 + -1, // value=-1 + -10, // value=-11 + -100, // value=-111 + -1000000, // value=-1000111 + 1000111, // value=0 + Number.MAX_SAFE_INTEGER, // value=9007199254740991 + // do next decrements in 2 steps as opposed to multiplying by -2 to prevent overflow + -Number.MAX_SAFE_INTEGER, // value=0 + -Number.MAX_SAFE_INTEGER, // value=-9007199254740991 + ]; + + // send increments one at a time and check expected value + for (let i = 0; i < increments.length; i++) { + const increment = increments[i]; + + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.counterIncOp({ + objectId: counterObjectId, + amount: increment, + }), + ); + expectedCounterValue += increment; + + expect(counter.value()).to.equal( + expectedCounterValue, + `Check counter at "${counterKey}" key in root has correct value after ${i + 1} COUNTER_INC ops`, + ); + } + }, + }, + ]; + + for (const scenario of applyOperationsScenarios) { + if (scenario.skip === true) { + continue; + } + + /** @nospec */ + it(`can apply ${scenario.description} state operation messages`, async function () { + const helper = this.test.helper; + const liveObjectsHelper = new LiveObjectsHelper(helper); + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channelName = `channel_can_apply_${scenario.description}`; + const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + await scenario.action({ root, liveObjectsHelper, channelName }); + }, client); + }); + } }); /** @nospec */ From 396085263e2d44e694d61359f19b71e402c718f4 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 22 Oct 2024 10:09:20 +0100 Subject: [PATCH 046/166] Update `moduleReport` config with latest bundle info --- scripts/moduleReport.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index e260f0783d..083f18ab9b 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -6,7 +6,7 @@ import { gzip } from 'zlib'; import Table from 'cli-table'; // The maximum size we allow for a minimal useful Realtime bundle (i.e. one that can subscribe to a channel) -const minimalUsefulRealtimeBundleSizeThresholdsKiB = { raw: 100, gzip: 31 }; +const minimalUsefulRealtimeBundleSizeThresholdsKiB = { raw: 101, gzip: 31 }; const baseClientNames = ['BaseRest', 'BaseRealtime']; @@ -310,12 +310,15 @@ async function checkLiveObjectsPluginFiles() { // These are the files that are allowed to contribute >= `threshold` bytes to the LiveObjects bundle. const allowedFiles = new Set([ 'src/plugins/liveobjects/index.ts', + 'src/plugins/liveobjects/livecounter.ts', 'src/plugins/liveobjects/livemap.ts', 'src/plugins/liveobjects/liveobject.ts', 'src/plugins/liveobjects/liveobjects.ts', 'src/plugins/liveobjects/liveobjectspool.ts', + 'src/plugins/liveobjects/objectid.ts', 'src/plugins/liveobjects/statemessage.ts', 'src/plugins/liveobjects/syncliveobjectsdatapool.ts', + 'src/plugins/liveobjects/timeserial.ts', ]); return checkBundleFiles(pluginBundleInfo, allowedFiles, 100); From 3ed285d2d31a12ab6112cd366ef3e5ab5d864df9 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 25 Oct 2024 08:36:28 +0100 Subject: [PATCH 047/166] Add `zeroValue` static method to LiveMap and LiveCounter --- src/plugins/liveobjects/livecounter.ts | 9 +++++++++ src/plugins/liveobjects/livemap.ts | 9 +++++++++ src/plugins/liveobjects/liveobjectspool.ts | 14 +++++++------- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index b82b9f852b..a8c3db6833 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -16,6 +16,15 @@ export class LiveCounter extends LiveObject { super(liveObjects, initialData, objectId); } + /** + * Returns a {@link LiveCounter} instance with a 0 value. + * + * @internal + */ + static zeroValue(liveobjects: LiveObjects, isCreated: boolean, objectId?: string): LiveCounter { + return new LiveCounter(liveobjects, isCreated, null, objectId); + } + value(): number { return this._dataRef.data; } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 90582d3a7c..2b8c27c550 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -50,6 +50,15 @@ export class LiveMap extends LiveObject { super(liveObjects, initialData, objectId); } + /** + * Returns a {@link LiveMap} instance with an empty map data. + * + * @internal + */ + static zeroValue(liveobjects: LiveObjects, objectId?: string): LiveMap { + return new LiveMap(liveobjects, MapSemantics.LWW, null, objectId); + } + static liveMapDataFromMapEntries(client: BaseClient, entries: Record): LiveMapData { const liveMapData: LiveMapData = { data: new Map(), diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index 02123f17c8..6db02c78e8 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -54,12 +54,12 @@ export class LiveObjectsPool { let zeroValueObject: LiveObject; switch (parsedObjectId.type) { case 'map': { - zeroValueObject = new LiveMap(this._liveObjects, MapSemantics.LWW, null, objectId); + zeroValueObject = LiveMap.zeroValue(this._liveObjects, objectId); break; } case 'counter': - zeroValueObject = new LiveCounter(this._liveObjects, false, null, objectId); + zeroValueObject = LiveCounter.zeroValue(this._liveObjects, false, objectId); break; } @@ -125,7 +125,7 @@ export class LiveObjectsPool { private _getInitialPool(): Map { const pool = new Map(); - const root = new LiveMap(this._liveObjects, MapSemantics.LWW, null, ROOT_OBJECT_ID); + const root = LiveMap.zeroValue(this._liveObjects, ROOT_OBJECT_ID); pool.set(root.getObjectId(), root); return pool; } @@ -133,8 +133,8 @@ export class LiveObjectsPool { private _handleCounterCreate(stateOperation: StateOperation): void { let counter: LiveCounter; if (this._client.Utils.isNil(stateOperation.counter)) { - // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly 0 in this case. - counter = new LiveCounter(this._liveObjects, true, { data: 0 }, stateOperation.objectId); + // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly a zero-value counter. + counter = LiveCounter.zeroValue(this._liveObjects, true, stateOperation.objectId); } else { counter = new LiveCounter( this._liveObjects, @@ -150,8 +150,8 @@ export class LiveObjectsPool { private _handleMapCreate(stateOperation: StateOperation): void { let map: LiveMap; if (this._client.Utils.isNil(stateOperation.map)) { - // if a map object is missing for the MAP_CREATE op, the initial value is implicitly an empty map. - map = new LiveMap(this._liveObjects, MapSemantics.LWW, null, stateOperation.objectId); + // if a map object is missing for the MAP_CREATE op, the initial value is implicitly a zero-value map. + map = LiveMap.zeroValue(this._liveObjects, stateOperation.objectId); } else { const objectData = LiveMap.liveMapDataFromMapEntries(this._client, stateOperation.map.entries ?? {}); map = new LiveMap( From b988c35ec5e39e0489a773a27ca19edf2b376a75 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 30 Oct 2024 11:38:20 +0000 Subject: [PATCH 048/166] Fix LiveObjects tests fail due to class name changes with static properties --- test/realtime/live_objects.test.js | 51 +++++++++++++++--------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index c0da311f30..885760acfe 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -24,6 +24,12 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }; } + function expectInstanceOf(object, className, msg) { + // esbuild changes the name for classes with static method to include an underscore as prefix. + // so LiveMap becomes _LiveMap. we account for it here. + expect(object.constructor.name).to.match(new RegExp(`_?${className}`), msg); + } + describe('realtime/live_objects', function () { this.timeout(60 * 1000); @@ -146,7 +152,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const helper = this.test.helper; const client = RealtimeWithLiveObjects(helper, { autoConnect: false }); const channel = client.channels.get('channel'); - expect(channel.liveObjects.constructor.name).to.equal('LiveObjects'); + expectInstanceOf(channel.liveObjects, 'LiveObjects'); }); /** @nospec */ @@ -161,7 +167,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await channel.attach(); const root = await liveObjects.getRoot(); - expect(root.constructor.name).to.equal('LiveMap'); + expectInstanceOf(root, 'LiveMap', 'root object should be of LiveMap type'); }, client); }); @@ -178,7 +184,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const root = await liveObjects.getRoot(); helper.recordPrivateApi('call.LiveObject.getObjectId'); - expect(root.getObjectId()).to.equal('root'); + expect(root.getObjectId()).to.equal('root', 'root object should have an object id "root"'); }, client); }); @@ -352,16 +358,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], counterKeys.forEach((key) => { const counter = root.get(key); expect(counter, `Check counter at key="${key}" in root exists`).to.exist; - expect(counter.constructor.name).to.equal( - 'LiveCounter', - `Check counter at key="${key}" in root is of type LiveCounter`, - ); + expectInstanceOf(counter, 'LiveCounter', `Check counter at key="${key}" in root is of type LiveCounter`); }); mapKeys.forEach((key) => { const map = root.get(key); expect(map, `Check map at key="${key}" in root exists`).to.exist; - expect(map.constructor.name).to.equal('LiveMap', `Check map at key="${key}" in root is of type LiveMap`); + expectInstanceOf(map, 'LiveMap', `Check map at key="${key}" in root is of type LiveMap`); }); const valuesMap = root.get('valuesMap'); @@ -474,10 +477,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const counterFromReferencedMap = referencedMap.get('counterKey'); expect(counterFromReferencedMap, 'Check nested counter exists at a key in a map').to.exist; - expect(counterFromReferencedMap.constructor.name).to.equal( - 'LiveCounter', - 'Check nested counter is of type LiveCounter', - ); + expectInstanceOf(counterFromReferencedMap, 'LiveCounter', 'Check nested counter is of type LiveCounter'); expect(counterFromReferencedMap).to.equal( referencedCounter, 'Check nested counter is the same object instance as counter on the root', @@ -486,7 +486,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const mapFromValuesMap = valuesMap.get('mapKey'); expect(mapFromValuesMap, 'Check nested map exists at a key in a map').to.exist; - expect(mapFromValuesMap.constructor.name).to.equal('LiveMap', 'Check nested map is of type LiveMap'); + expectInstanceOf(mapFromValuesMap, 'LiveMap', 'Check nested map is of type LiveMap'); expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); expect(mapFromValuesMap).to.equal( referencedMap, @@ -561,10 +561,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check all maps exist on root expect(mapObj, `Check map at "${key}" key in root exists`).to.exist; - expect(mapObj.constructor.name).to.equal( - 'LiveMap', - `Check map at "${key}" key in root is of type LiveMap`, - ); + expectInstanceOf(mapObj, 'LiveMap', `Check map at "${key}" key in root is of type LiveMap`); // check primitive maps have correct values expect(mapObj.size()).to.equal( @@ -630,7 +627,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check map with references exist on root const withReferencesMap = root.get(withReferencesMapKey); expect(withReferencesMap, `Check map at "${withReferencesMapKey}" key in root exists`).to.exist; - expect(withReferencesMap.constructor.name).to.equal( + expectInstanceOf( + withReferencesMap, 'LiveMap', `Check map at "${withReferencesMapKey}" key in root is of type LiveMap`, ); @@ -645,17 +643,16 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const referencedMap = withReferencesMap.get('mapReference'); expect(referencedCounter, `Check counter at "counterReference" exists`).to.exist; - expect(referencedCounter.constructor.name).to.equal( + expectInstanceOf( + referencedCounter, 'LiveCounter', `Check counter at "counterReference" key is of type LiveCounter`, ); expect(referencedCounter.value()).to.equal(1, 'Check counter at "counterReference" key has correct value'); expect(referencedMap, `Check map at "mapReference" key exists`).to.exist; - expect(referencedMap.constructor.name).to.equal( - 'LiveMap', - `Check map at "mapReference" key is of type LiveMap`, - ); + expectInstanceOf(referencedMap, 'LiveMap', `Check map at "mapReference" key is of type LiveMap`); + expect(referencedMap.size()).to.equal(1, 'Check map at "mapReference" key has correct number of keys'); expect(referencedMap.get('stringKey')).to.equal( 'stringValue', @@ -741,14 +738,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const map = root.get('keyToMap'); expect(counter, 'Check counter at "keyToCounter" key in root exists').to.exist; - expect(counter.constructor.name).to.equal( + expectInstanceOf( + counter, 'LiveCounter', 'Check counter at "keyToCounter" key in root is of type LiveCounter', ); expect(counter.value()).to.equal(1, 'Check counter at "keyToCounter" key in root has correct value'); expect(map, 'Check map at "keyToMap" key in root exists').to.exist; - expect(map.constructor.name).to.equal('LiveMap', 'Check map at "keyToMap" key in root is of type LiveMap'); + expectInstanceOf(map, 'LiveMap', 'Check map at "keyToMap" key in root is of type LiveMap'); expect(map.size()).to.equal(1, 'Check map at "keyToMap" key in root has correct number of keys'); expect(map.get('stringKey')).to.equal( 'stringValue', @@ -849,7 +847,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check all counters exist on root expect(counterObj, `Check counter at "${key}" key in root exists`).to.exist; - expect(counterObj.constructor.name).to.equal( + expectInstanceOf( + counterObj, 'LiveCounter', `Check counter at "${key}" key in root is of type LiveCounter`, ); From 0a39b233060b345b7b831095ca4187e569ff8740 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 24 Oct 2024 13:21:48 +0100 Subject: [PATCH 049/166] Allow `seriesId` to be empty in Timeserial This allows parsing of zero value `@0-0` timeserials. --- src/plugins/liveobjects/timeserial.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/liveobjects/timeserial.ts b/src/plugins/liveobjects/timeserial.ts index 553f960a50..05d98e4b95 100644 --- a/src/plugins/liveobjects/timeserial.ts +++ b/src/plugins/liveobjects/timeserial.ts @@ -78,7 +78,7 @@ export class DefaultTimeserial implements Timeserial { } const [seriesId, rest] = timeserial.split('@'); - if (!seriesId || !rest) { + if (!rest) { throw new client.ErrorInfo(`Invalid timeserial: ${timeserial}`, 50000, 500); } From 60e6474bb5b138f5951a4b4688d663948f8cd58e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 24 Oct 2024 18:41:50 +0100 Subject: [PATCH 050/166] Change seriesId comparison logic for Timeserial Revert the change made in aacf17d73e759365ed477ea4439cbe37db67e7ea where an empty seriesId was considered less than a non-empty one, and change the comparison logic to match the one on the realtime side [1]. [1] https://github.com/ably/realtime/blob/c8fc68f6a6be21fefc42b2009272bea1ecefbbcb/nodejs/realtime/common/lib/types/timeseries.ts#L134-L139 --- src/plugins/liveobjects/timeserial.ts | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/plugins/liveobjects/timeserial.ts b/src/plugins/liveobjects/timeserial.ts index 05d98e4b95..4d764a6935 100644 --- a/src/plugins/liveobjects/timeserial.ts +++ b/src/plugins/liveobjects/timeserial.ts @@ -125,20 +125,14 @@ export class DefaultTimeserial implements Timeserial { return counterDiff; } - // Compare the seriesId - // An empty seriesId is considered less than a non-empty one - if (!this.seriesId && secondTimeserial.seriesId) { - return -1; - } - if (this.seriesId && !secondTimeserial.seriesId) { - return 1; - } - // Otherwise compare seriesId lexicographically - const seriesIdDiff = - this.seriesId === secondTimeserial.seriesId ? 0 : this.seriesId < secondTimeserial.seriesId ? -1 : 1; - - if (seriesIdDiff) { - return seriesIdDiff; + // Compare the seriesId lexicographically, but only if both seriesId exist + const seriesComparison = + this.seriesId && + secondTimeserial.seriesId && + this.seriesId !== secondTimeserial.seriesId && + (this.seriesId > secondTimeserial.seriesId ? 1 : -1); + if (seriesComparison) { + return seriesComparison; } // Compare the index, if present From 65c3f0f9385d9862940ea21b4c7fad26104e80aa Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 24 Oct 2024 21:44:49 +0100 Subject: [PATCH 051/166] Use zero-value timeserial if timeserial is missing for MapEntry in a MAP_CREATE operation --- src/plugins/liveobjects/livemap.ts | 34 +++++++++++++------------ src/plugins/liveobjects/statemessage.ts | 9 +++++-- src/plugins/liveobjects/timeserial.ts | 9 +++++++ 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 2b8c27c550..f447795aa7 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -75,7 +75,9 @@ export class LiveMap extends LiveObject { const liveDataEntry: MapEntry = { ...entry, - timeserial: DefaultTimeserial.calculateTimeserial(client, entry.timeserial), + timeserial: entry.timeserial + ? DefaultTimeserial.calculateTimeserial(client, entry.timeserial) + : DefaultTimeserial.zeroValueTimeserial(client), // true only if we received explicit true. otherwise always false tombstone: entry.tombstone === true, data: liveData, @@ -150,7 +152,7 @@ export class LiveMap extends LiveObject { if (this._client.Utils.isNil(op.mapOp)) { this._throwNoPayloadError(op); } else { - this._applyMapSet(op.mapOp, msg.serial); + this._applyMapSet(op.mapOp, DefaultTimeserial.calculateTimeserial(this._client, msg.serial)); } break; @@ -158,7 +160,7 @@ export class LiveMap extends LiveObject { if (this._client.Utils.isNil(op.mapOp)) { this._throwNoPayloadError(op); } else { - this._applyMapRemove(op.mapOp, msg.serial); + this._applyMapRemove(op.mapOp, DefaultTimeserial.calculateTimeserial(this._client, msg.serial)); } break; @@ -202,7 +204,9 @@ export class LiveMap extends LiveObject { // we can do this by iterating over entries from MAP_CREATE op and apply changes on per-key basis as if we had MAP_SET, MAP_REMOVE operations. Object.entries(op.entries ?? {}).forEach(([key, entry]) => { // for MAP_CREATE op we must use dedicated timeserial field available on an entry, instead of a timeserial on a message - const opOriginTimeserial = entry.timeserial; + const opOriginTimeserial = entry.timeserial + ? DefaultTimeserial.calculateTimeserial(this._client, entry.timeserial) + : DefaultTimeserial.zeroValueTimeserial(this._client); if (entry.tombstone === true) { // entry in MAP_CREATE op is deleted, try to apply MAP_REMOVE op this._applyMapRemove({ key }, opOriginTimeserial); @@ -213,18 +217,17 @@ export class LiveMap extends LiveObject { }); } - private _applyMapSet(op: StateMapOp, opOriginTimeserialStr: string | undefined): void { + private _applyMapSet(op: StateMapOp, opOriginTimeserial: Timeserial): void { const { ErrorInfo, Utils } = this._client; - const opTimeserial = DefaultTimeserial.calculateTimeserial(this._client, opOriginTimeserialStr); const existingEntry = this._dataRef.data.get(op.key); - if (existingEntry && opTimeserial.before(existingEntry.timeserial)) { + if (existingEntry && opOriginTimeserial.before(existingEntry.timeserial)) { // the operation's origin timeserial < the entry's timeserial, ignore the operation. this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MICRO, 'LiveMap._applyMapSet()', - `skipping updating key="${op.key}" as existing key entry has greater timeserial: ${existingEntry.timeserial.toString()}, than the op: ${opOriginTimeserialStr}; objectId=${this._objectId}`, + `skipping updating key="${op.key}" as existing key entry has greater timeserial: ${existingEntry.timeserial.toString()}, than the op: ${opOriginTimeserial.toString()}; objectId=${this._objectId}`, ); return; } @@ -251,40 +254,39 @@ export class LiveMap extends LiveObject { if (existingEntry) { existingEntry.tombstone = false; - existingEntry.timeserial = opTimeserial; + existingEntry.timeserial = opOriginTimeserial; existingEntry.data = liveData; } else { const newEntry: MapEntry = { tombstone: false, - timeserial: opTimeserial, + timeserial: opOriginTimeserial, data: liveData, }; this._dataRef.data.set(op.key, newEntry); } } - private _applyMapRemove(op: StateMapOp, opOriginTimeserialStr: string | undefined): void { - const opTimeserial = DefaultTimeserial.calculateTimeserial(this._client, opOriginTimeserialStr); + private _applyMapRemove(op: StateMapOp, opOriginTimeserial: Timeserial): void { const existingEntry = this._dataRef.data.get(op.key); - if (existingEntry && opTimeserial.before(existingEntry.timeserial)) { + if (existingEntry && opOriginTimeserial.before(existingEntry.timeserial)) { // the operation's origin timeserial < the entry's timeserial, ignore the operation. this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MICRO, 'LiveMap._applyMapRemove()', - `skipping removing key="${op.key}" as existing key entry has greater timeserial: ${existingEntry.timeserial.toString()}, than the op: ${opOriginTimeserialStr}; objectId=${this._objectId}`, + `skipping removing key="${op.key}" as existing key entry has greater timeserial: ${existingEntry.timeserial.toString()}, than the op: ${opOriginTimeserial.toString()}; objectId=${this._objectId}`, ); return; } if (existingEntry) { existingEntry.tombstone = true; - existingEntry.timeserial = opTimeserial; + existingEntry.timeserial = opOriginTimeserial; existingEntry.data = undefined; } else { const newEntry: MapEntry = { tombstone: true, - timeserial: opTimeserial, + timeserial: opOriginTimeserial, data: undefined, }; this._dataRef.data.set(op.key, newEntry); diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 2fd293a336..b4eefdadd0 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -48,8 +48,13 @@ export interface StateCounterOp { export interface StateMapEntry { /** Indicates whether the map entry has been removed. */ tombstone?: boolean; - /** The *origin* timeserial of the last operation that was applied to the map entry. */ - timeserial: string; + /** + * The *origin* timeserial of the last operation that was applied to the map entry. + * + * It is optional in a MAP_CREATE operation and might be missing, in which case the client should default to using zero-value timeserial, + * which is the "earliest possible" timeserial. This will allow any other operation to update the field based on a timeserial comparison. + */ + timeserial?: string; /** The data that represents the value of the map entry. */ data: StateData; } diff --git a/src/plugins/liveobjects/timeserial.ts b/src/plugins/liveobjects/timeserial.ts index 4d764a6935..0accdc81be 100644 --- a/src/plugins/liveobjects/timeserial.ts +++ b/src/plugins/liveobjects/timeserial.ts @@ -101,6 +101,15 @@ export class DefaultTimeserial implements Timeserial { ); } + /** + * Returns a zero-value Timeserial `@0-0` - "earliest possible" timeserial. + * + * @returns The timeserial object. + */ + static zeroValueTimeserial(client: BaseClient): Timeserial { + return new DefaultTimeserial(client, '', 0, 0); // @0-0 + } + /** * Compares this timeserial to the supplied timeserial, returning a number indicating their relative order. * @param timeserialToCompare The timeserial to compare against. Can be a string or a Timeserial object. From 7ccfe50f518d490374d2ced160071fa9072110a1 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 24 Oct 2024 14:03:01 +0100 Subject: [PATCH 052/166] Handle regional timeserials for LiveObjects Regional timeserial for a LiveObject: - is set to StateObject.regionalTimeserial when object is created during SYNC sequence - is set to message's channelSerial property when object is created via a state operation message - is updated to message's channelSerial property when an operation is applied on an object via a state operation message - is equal to zero-value Timeserial (`@0-0`) when creating a zero-value object --- src/common/lib/client/realtimechannel.ts | 2 +- src/plugins/liveobjects/livecounter.ts | 19 +++++++++++---- src/plugins/liveobjects/livemap.ts | 11 +++++---- src/plugins/liveobjects/liveobject.ts | 12 ++++++---- src/plugins/liveobjects/liveobjects.ts | 14 ++++++----- src/plugins/liveobjects/liveobjectspool.ts | 27 +++++++++++++--------- 6 files changed, 54 insertions(+), 31 deletions(-) diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index 4cdcd23126..b7ea4e577a 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -649,7 +649,7 @@ class RealtimeChannel extends EventEmitter { } } - this._liveObjects.handleStateMessages(stateMessages); + this._liveObjects.handleStateMessages(stateMessages, message.channelSerial); break; } diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index a8c3db6833..61d17d9a24 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -1,6 +1,7 @@ import { LiveObject, LiveObjectData } from './liveobject'; import { LiveObjects } from './liveobjects'; -import { StateCounter, StateCounterOp, StateOperation, StateOperationAction } from './statemessage'; +import { StateCounter, StateCounterOp, StateMessage, StateOperation, StateOperationAction } from './statemessage'; +import { Timeserial } from './timeserial'; export interface LiveCounterData extends LiveObjectData { data: number; @@ -12,8 +13,9 @@ export class LiveCounter extends LiveObject { private _created: boolean, initialData?: LiveCounterData | null, objectId?: string, + regionalTimeserial?: Timeserial, ) { - super(liveObjects, initialData, objectId); + super(liveObjects, initialData, objectId, regionalTimeserial); } /** @@ -21,8 +23,13 @@ export class LiveCounter extends LiveObject { * * @internal */ - static zeroValue(liveobjects: LiveObjects, isCreated: boolean, objectId?: string): LiveCounter { - return new LiveCounter(liveobjects, isCreated, null, objectId); + static zeroValue( + liveobjects: LiveObjects, + isCreated: boolean, + objectId?: string, + regionalTimeserial?: Timeserial, + ): LiveCounter { + return new LiveCounter(liveobjects, isCreated, null, objectId, regionalTimeserial); } value(): number { @@ -46,7 +53,7 @@ export class LiveCounter extends LiveObject { /** * @internal */ - applyOperation(op: StateOperation): void { + applyOperation(op: StateOperation, msg: StateMessage, opRegionalTimeserial: Timeserial): void { if (op.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( `Cannot apply state operation with objectId=${op.objectId}, to this LiveCounter with objectId=${this.getObjectId()}`, @@ -75,6 +82,8 @@ export class LiveCounter extends LiveObject { 500, ); } + + this.setRegionalTimeserial(opRegionalTimeserial); } protected _getZeroValueData(): LiveCounterData { diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index f447795aa7..42d8248e7d 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -46,8 +46,9 @@ export class LiveMap extends LiveObject { private _semantics: MapSemantics, initialData?: LiveMapData | null, objectId?: string, + regionalTimeserial?: Timeserial, ) { - super(liveObjects, initialData, objectId); + super(liveObjects, initialData, objectId, regionalTimeserial); } /** @@ -55,8 +56,8 @@ export class LiveMap extends LiveObject { * * @internal */ - static zeroValue(liveobjects: LiveObjects, objectId?: string): LiveMap { - return new LiveMap(liveobjects, MapSemantics.LWW, null, objectId); + static zeroValue(liveobjects: LiveObjects, objectId?: string, regionalTimeserial?: Timeserial): LiveMap { + return new LiveMap(liveobjects, MapSemantics.LWW, null, objectId, regionalTimeserial); } static liveMapDataFromMapEntries(client: BaseClient, entries: Record): LiveMapData { @@ -134,7 +135,7 @@ export class LiveMap extends LiveObject { /** * @internal */ - applyOperation(op: StateOperation, msg: StateMessage): void { + applyOperation(op: StateOperation, msg: StateMessage, opRegionalTimeserial: Timeserial): void { if (op.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( `Cannot apply state operation with objectId=${op.objectId}, to this LiveMap with objectId=${this.getObjectId()}`, @@ -171,6 +172,8 @@ export class LiveMap extends LiveObject { 500, ); } + + this.setRegionalTimeserial(opRegionalTimeserial); } protected _getZeroValueData(): LiveMapData { diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index 2e33cb0b23..70a294779b 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -1,6 +1,7 @@ import type BaseClient from 'common/lib/client/baseclient'; import { LiveObjects } from './liveobjects'; import { StateMessage, StateOperation } from './statemessage'; +import { DefaultTimeserial, Timeserial } from './timeserial'; export interface LiveObjectData { data: any; @@ -10,16 +11,19 @@ export abstract class LiveObject { protected _client: BaseClient; protected _dataRef: T; protected _objectId: string; - protected _regionalTimeserial?: string; + protected _regionalTimeserial: Timeserial; constructor( protected _liveObjects: LiveObjects, initialData?: T | null, objectId?: string, + regionalTimeserial?: Timeserial, ) { this._client = this._liveObjects.getClient(); this._dataRef = initialData ?? this._getZeroValueData(); this._objectId = objectId ?? this._createObjectId(); + // use zero value timeserial by default, so any future operation can be applied for this object + this._regionalTimeserial = regionalTimeserial ?? DefaultTimeserial.zeroValueTimeserial(this._client); } /** @@ -32,7 +36,7 @@ export abstract class LiveObject { /** * @internal */ - getRegionalTimeserial(): string | undefined { + getRegionalTimeserial(): Timeserial { return this._regionalTimeserial; } @@ -46,7 +50,7 @@ export abstract class LiveObject { /** * @internal */ - setRegionalTimeserial(regionalTimeserial: string): void { + setRegionalTimeserial(regionalTimeserial: Timeserial): void { this._regionalTimeserial = regionalTimeserial; } @@ -58,6 +62,6 @@ export abstract class LiveObject { /** * @internal */ - abstract applyOperation(op: StateOperation, msg: StateMessage): void; + abstract applyOperation(op: StateOperation, msg: StateMessage, opRegionalTimeserial: Timeserial): void; protected abstract _getZeroValueData(): T; } diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 0fb0886ed1..7a80c2de9b 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -8,6 +8,7 @@ import { LiveObject } from './liveobject'; import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage } from './statemessage'; import { LiveCounterDataEntry, SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; +import { DefaultTimeserial } from './timeserial'; enum LiveObjectsEvents { SyncCompleted = 'SyncCompleted', @@ -84,12 +85,13 @@ export class LiveObjects { /** * @internal */ - handleStateMessages(stateMessages: StateMessage[]): void { + handleStateMessages(stateMessages: StateMessage[], msgRegionalTimeserial: string | null | undefined): void { + const timeserial = DefaultTimeserial.calculateTimeserial(this._client, msgRegionalTimeserial); if (this._syncInProgress) { // TODO: handle buffering of state messages during SYNC } - this._liveObjectsPool.applyStateMessages(stateMessages); + this._liveObjectsPool.applyStateMessages(stateMessages, timeserial); } /** @@ -180,10 +182,11 @@ export class LiveObjects { for (const [objectId, entry] of this._syncLiveObjectsDataPool.entries()) { receivedObjectIds.add(objectId); const existingObject = this._liveObjectsPool.get(objectId); + const regionalTimeserialObj = DefaultTimeserial.calculateTimeserial(this._client, entry.regionalTimeserial); if (existingObject) { existingObject.setData(entry.objectData); - existingObject.setRegionalTimeserial(entry.regionalTimeserial); + existingObject.setRegionalTimeserial(regionalTimeserialObj); if (existingObject instanceof LiveCounter) { existingObject.setCreated((entry as LiveCounterDataEntry).created); } @@ -195,17 +198,16 @@ export class LiveObjects { const objectType = entry.objectType; switch (objectType) { case 'LiveCounter': - newObject = new LiveCounter(this, entry.created, entry.objectData, objectId); + newObject = new LiveCounter(this, entry.created, entry.objectData, objectId, regionalTimeserialObj); break; case 'LiveMap': - newObject = new LiveMap(this, entry.semantics, entry.objectData, objectId); + newObject = new LiveMap(this, entry.semantics, entry.objectData, objectId, regionalTimeserialObj); break; default: throw new this._client.ErrorInfo(`Unknown live object type: ${objectType}`, 50000, 500); } - newObject.setRegionalTimeserial(entry.regionalTimeserial); this._liveObjectsPool.set(objectId, newObject); } diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index 6db02c78e8..c1466a8fcd 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -6,6 +6,7 @@ import { LiveObject } from './liveobject'; import { LiveObjects } from './liveobjects'; import { ObjectId } from './objectid'; import { MapSemantics, StateMessage, StateOperation, StateOperationAction } from './statemessage'; +import { DefaultTimeserial, Timeserial } from './timeserial'; export const ROOT_OBJECT_ID = 'root'; @@ -51,22 +52,24 @@ export class LiveObjectsPool { } const parsedObjectId = ObjectId.fromString(this._client, objectId); + // use zero value timeserial, so any operation can be applied for this object + const regionalTimeserial = DefaultTimeserial.zeroValueTimeserial(this._client); let zeroValueObject: LiveObject; switch (parsedObjectId.type) { case 'map': { - zeroValueObject = LiveMap.zeroValue(this._liveObjects, objectId); + zeroValueObject = LiveMap.zeroValue(this._liveObjects, objectId, regionalTimeserial); break; } case 'counter': - zeroValueObject = LiveCounter.zeroValue(this._liveObjects, false, objectId); + zeroValueObject = LiveCounter.zeroValue(this._liveObjects, false, objectId, regionalTimeserial); break; } this.set(objectId, zeroValueObject); } - applyStateMessages(stateMessages: StateMessage[]): void { + applyStateMessages(stateMessages: StateMessage[], regionalTimeserial: Timeserial): void { for (const stateMessage of stateMessages) { if (!stateMessage.operation) { this._client.Logger.logAction( @@ -87,17 +90,17 @@ export class LiveObjectsPool { // object wich such id already exists (we may have created a zero-value object before, or this is a duplicate *_CREATE op), // so delegate application of the op to that object // TODO: invoke subscription callbacks for an object when applied - this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); + this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage, regionalTimeserial); break; } // otherwise we can create new objects in the pool if (stateOperation.action === StateOperationAction.MAP_CREATE) { - this._handleMapCreate(stateOperation); + this._handleMapCreate(stateOperation, regionalTimeserial); } if (stateOperation.action === StateOperationAction.COUNTER_CREATE) { - this._handleCounterCreate(stateOperation); + this._handleCounterCreate(stateOperation, regionalTimeserial); } break; @@ -109,7 +112,7 @@ export class LiveObjectsPool { // when we eventually receive a corresponding *_CREATE op for that object, its application will be handled by that zero-value object. this.createZeroValueObjectIfNotExists(stateOperation.objectId); // TODO: invoke subscription callbacks for an object when applied - this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); + this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage, regionalTimeserial); break; default: @@ -130,28 +133,29 @@ export class LiveObjectsPool { return pool; } - private _handleCounterCreate(stateOperation: StateOperation): void { + private _handleCounterCreate(stateOperation: StateOperation, opRegionalTimeserial: Timeserial): void { let counter: LiveCounter; if (this._client.Utils.isNil(stateOperation.counter)) { // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly a zero-value counter. - counter = LiveCounter.zeroValue(this._liveObjects, true, stateOperation.objectId); + counter = LiveCounter.zeroValue(this._liveObjects, true, stateOperation.objectId, opRegionalTimeserial); } else { counter = new LiveCounter( this._liveObjects, true, { data: stateOperation.counter.count ?? 0 }, stateOperation.objectId, + opRegionalTimeserial, ); } this.set(stateOperation.objectId, counter); } - private _handleMapCreate(stateOperation: StateOperation): void { + private _handleMapCreate(stateOperation: StateOperation, opRegionalTimeserial: Timeserial): void { let map: LiveMap; if (this._client.Utils.isNil(stateOperation.map)) { // if a map object is missing for the MAP_CREATE op, the initial value is implicitly a zero-value map. - map = LiveMap.zeroValue(this._liveObjects, stateOperation.objectId); + map = LiveMap.zeroValue(this._liveObjects, stateOperation.objectId, opRegionalTimeserial); } else { const objectData = LiveMap.liveMapDataFromMapEntries(this._client, stateOperation.map.entries ?? {}); map = new LiveMap( @@ -159,6 +163,7 @@ export class LiveObjectsPool { stateOperation.map.semantics ?? MapSemantics.LWW, objectData, stateOperation.objectId, + opRegionalTimeserial, ); } From ad0311e252fd83f2524c88159586d73a790984c1 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 24 Oct 2024 14:11:18 +0100 Subject: [PATCH 053/166] Implement buffering and application of operations during a SYNC sequence - state operation messages are buffered while SYNC is in progress - all buffered operations are discarded when new SYNC starts - when SYNC ends operations to apply are decided based on the regional timeserial of the message - eligible operations are applied via a regular LiveObject operation application logic Resolves DTP-955 --- src/plugins/liveobjects/liveobjects.ts | 31 +++++++++++++--- src/plugins/liveobjects/liveobjectspool.ts | 42 +++++++++++++++++++++- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 7a80c2de9b..9b1981f63a 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -8,12 +8,17 @@ import { LiveObject } from './liveobject'; import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage } from './statemessage'; import { LiveCounterDataEntry, SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; -import { DefaultTimeserial } from './timeserial'; +import { DefaultTimeserial, Timeserial } from './timeserial'; enum LiveObjectsEvents { SyncCompleted = 'SyncCompleted', } +export interface BufferedStateMessage { + stateMessage: StateMessage; + regionalTimeserial: Timeserial; +} + export class LiveObjects { private _client: BaseClient; private _channel: RealtimeChannel; @@ -25,6 +30,7 @@ export class LiveObjects { private _syncInProgress: boolean; private _currentSyncId: string | undefined; private _currentSyncCursor: string | undefined; + private _bufferedStateOperations: BufferedStateMessage[]; constructor(channel: RealtimeChannel) { this._channel = channel; @@ -33,6 +39,7 @@ export class LiveObjects { this._liveObjectsPool = new LiveObjectsPool(this); this._syncLiveObjectsDataPool = new SyncLiveObjectsDataPool(this); this._syncInProgress = true; + this._bufferedStateOperations = []; } async getRoot(): Promise { @@ -87,8 +94,18 @@ export class LiveObjects { */ handleStateMessages(stateMessages: StateMessage[], msgRegionalTimeserial: string | null | undefined): void { const timeserial = DefaultTimeserial.calculateTimeserial(this._client, msgRegionalTimeserial); + if (this._syncInProgress) { - // TODO: handle buffering of state messages during SYNC + // The client receives state messages in realtime over the channel concurrently with the SYNC sequence. + // Some of the incoming state messages may have already been applied to the state objects described in + // the SYNC sequence, but others may not; therefore we must buffer these messages so that we can apply + // them to the state objects once the SYNC is complete. To avoid double-counting, the buffered operations + // are applied according to the state object's regional timeserial, which reflects the regional timeserial + // of the state message that was last applied to that state object. + stateMessages.forEach((x) => + this._bufferedStateOperations.push({ stateMessage: x, regionalTimeserial: timeserial }), + ); + return; } this._liveObjectsPool.applyStateMessages(stateMessages, timeserial); @@ -102,7 +119,7 @@ export class LiveObjects { this._client.logger, this._client.Logger.LOG_MINOR, 'LiveObjects.onAttached()', - 'channel = ' + this._channel.name + ', hasState = ' + hasState, + `channel=${this._channel.name}, hasState=${hasState}`, ); if (hasState) { @@ -137,6 +154,8 @@ export class LiveObjects { } private _startNewSync(syncId?: string, syncCursor?: string): void { + // need to discard all buffered state operation messages on new sync start + this._bufferedStateOperations = []; this._syncLiveObjectsDataPool.reset(); this._currentSyncId = syncId; this._currentSyncCursor = syncCursor; @@ -144,9 +163,11 @@ export class LiveObjects { } private _endSync(): void { - // TODO: handle applying buffered state messages when SYNC is finished - this._applySync(); + // should apply buffered state operations after we applied the SYNC data + this._liveObjectsPool.applyBufferedStateMessages(this._bufferedStateOperations); + + this._bufferedStateOperations = []; this._syncLiveObjectsDataPool.reset(); this._currentSyncId = undefined; this._currentSyncCursor = undefined; diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index c1466a8fcd..3d41e2e238 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -3,7 +3,7 @@ import type RealtimeChannel from 'common/lib/client/realtimechannel'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; -import { LiveObjects } from './liveobjects'; +import { BufferedStateMessage, LiveObjects } from './liveobjects'; import { ObjectId } from './objectid'; import { MapSemantics, StateMessage, StateOperation, StateOperationAction } from './statemessage'; import { DefaultTimeserial, Timeserial } from './timeserial'; @@ -126,6 +126,46 @@ export class LiveObjectsPool { } } + applyBufferedStateMessages(bufferedStateMessages: BufferedStateMessage[]): void { + // since we receive state operation messages concurrently with the SYNC sequence, + // we must determine which operation messages should be applied to the now local copy of the object pool, and the rest will be skipped. + // since messages are delivered in regional order to the client, we can inspect the regional timeserial + // of each state operation message to know whether it has reached a point in the message stream + // that is no longer included in the state object snapshot we received from SYNC sequence. + for (const { regionalTimeserial, stateMessage } of bufferedStateMessages) { + if (!stateMessage.operation) { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MAJOR, + 'LiveObjects.LiveObjectsPool.applyBufferedStateMessages()', + `state operation message is received without 'operation' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + ); + continue; + } + + const existingObject = this.get(stateMessage.operation.objectId); + if (!existingObject) { + // for object ids we haven't seen yet we can apply operation immediately + this.applyStateMessages([stateMessage], regionalTimeserial); + continue; + } + + // otherwise we need to compare regional timeserials + if (regionalTimeserial.before(existingObject.getRegionalTimeserial())) { + // the operation's regional timeserial < the object's timeserial, ignore the operation. + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveObjects.LiveObjectsPool.applyBufferedStateMessages()', + `skipping applying buffered state operation message as existing object has greater regional timeserial: ${existingObject.getRegionalTimeserial().toString()}, than the op: ${regionalTimeserial.toString()}; objectId=${stateMessage.operation.objectId}, message id: ${stateMessage.id}, channel: ${this._channel.name}`, + ); + continue; + } + + this.applyStateMessages([stateMessage], regionalTimeserial); + } + } + private _getInitialPool(): Map { const pool = new Map(); const root = LiveMap.zeroValue(this._liveObjects, ROOT_OBJECT_ID); From a5f86670fbedea218a037cf446d2be3bd604487c Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 24 Oct 2024 18:19:12 +0100 Subject: [PATCH 054/166] Make injecting state messages in LiveObjects tests simpler This refactoring is needed for upcoming tests for buffering of state operation messages. --- test/common/modules/live_objects_helper.js | 96 ++++++++++++++++- test/realtime/live_objects.test.js | 116 +++++++-------------- 2 files changed, 131 insertions(+), 81 deletions(-) diff --git a/test/common/modules/live_objects_helper.js b/test/common/modules/live_objects_helper.js index b2793804e6..82c65b7377 100644 --- a/test/common/modules/live_objects_helper.js +++ b/test/common/modules/live_objects_helper.js @@ -3,7 +3,9 @@ /** * LiveObjects helper to create pre-determined state tree on channels */ -define(['shared_helper'], function (Helper) { +define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveObjectsPlugin) { + const createPM = Ably.makeProtocolMessageFromDeserialized({ LiveObjectsPlugin }); + const ACTIONS = { MAP_CREATE: 0, MAP_SET: 1, @@ -18,6 +20,7 @@ define(['shared_helper'], function (Helper) { class LiveObjectsHelper { constructor(helper) { + this._helper = helper; this._rest = helper.AblyRest({ useBinaryProtocol: false }); } @@ -171,6 +174,97 @@ define(['shared_helper'], function (Helper) { return op; } + mapObject(opts) { + const { objectId, regionalTimeserial, entries } = opts; + const obj = { + object: { + objectId, + regionalTimeserial, + map: { entries }, + }, + }; + + return obj; + } + + counterObject(opts) { + const { objectId, regionalTimeserial, count } = opts; + const obj = { + object: { + objectId, + regionalTimeserial, + counter: { + created: true, + count, + }, + }, + }; + + return obj; + } + + stateOperationMessage(opts) { + const { channelName, serial, state } = opts; + + state?.forEach((x, i) => (x.serial = `${serial}:${i}`)); + + return { + action: 19, // STATE + channel: channelName, + channelSerial: serial, + state: state ?? [], + }; + } + + stateObjectMessage(opts) { + const { channelName, syncSerial, state } = opts; + + return { + action: 20, // STATE_SYNC + channel: channelName, + channelSerial: syncSerial, + state: state ?? [], + }; + } + + async processStateOperationMessageOnChannel(opts) { + const { channel, ...rest } = opts; + + this._helper.recordPrivateApi('call.channel.processMessage'); + this._helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + await channel.processMessage( + createPM( + this.stateOperationMessage({ + ...rest, + channelName: channel.name, + }), + ), + ); + } + + async processStateObjectMessageOnChannel(opts) { + const { channel, ...rest } = opts; + + this._helper.recordPrivateApi('call.channel.processMessage'); + this._helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + await channel.processMessage( + createPM( + this.stateObjectMessage({ + ...rest, + channelName: channel.name, + }), + ), + ); + } + + fakeMapObjectId() { + return `map:${Helper.randomString()}`; + } + + fakeCounterObjectId() { + return `counter:${Helper.randomString()}`; + } + async stateRequest(channelName, opBody) { if (Array.isArray(opBody)) { throw new Error(`Only single object state requests are supported`); diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 885760acfe..0402e39dac 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -61,6 +61,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ it(`doesn't break when it receives a STATE ProtocolMessage`, async function () { const helper = this.test.helper; + const liveObjectsHelper = new LiveObjectsHelper(helper); const testClient = helper.AblyRealtime(); await helper.monitorConnectionThenCloseAndFinish(async () => { @@ -73,25 +74,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await helper.monitorConnectionThenCloseAndFinish(async () => { // inject STATE message that should be ignored and not break anything without LiveObjects plugin - helper.recordPrivateApi('call.channel.processMessage'); - helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); - await testChannel.processMessage( - createPM({ - action: 19, - channel: 'channel', - channelSerial: 'serial:', - state: [ - { - operation: { - action: 1, - objectId: 'root', - mapOp: { key: 'stringKey', data: { value: 'stringValue' } }, - }, - serial: 'a@0-0', - }, - ], - }), - ); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel: testChannel, + serial: '@0-0', + state: [ + liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'stringKey', data: { value: 'stringValue' } }), + ], + }); const publishChannel = publishClient.channels.get('channel'); await publishChannel.publish(null, 'test'); @@ -105,6 +94,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ it(`doesn't break when it receives a STATE_SYNC ProtocolMessage`, async function () { const helper = this.test.helper; + const liveObjectsHelper = new LiveObjectsHelper(helper); const testClient = helper.AblyRealtime(); await helper.monitorConnectionThenCloseAndFinish(async () => { @@ -117,24 +107,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await helper.monitorConnectionThenCloseAndFinish(async () => { // inject STATE_SYNC message that should be ignored and not break anything without LiveObjects plugin - helper.recordPrivateApi('call.channel.processMessage'); - helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); - await testChannel.processMessage( - createPM({ - action: 20, - channel: 'channel', - channelSerial: 'serial:', - state: [ - { - object: { - objectId: 'root', - regionalTimeserial: 'a@0-0', - map: {}, - }, - }, - ], - }), - ); + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel: testChannel, + syncSerial: 'serial:', + state: [liveObjectsHelper.mapObject({ objectId: 'root', regionalTimeserial: '@0-0' })], + }); const publishChannel = publishClient.channels.get('channel'); await publishChannel.publish(null, 'test'); @@ -261,6 +238,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ it('getRoot() waits for subsequent STATE_SYNC to finish before resolving', async function () { const helper = this.test.helper; + const liveObjectsHelper = new LiveObjectsHelper(helper); const client = RealtimeWithLiveObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { @@ -272,23 +250,17 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await liveObjects.getRoot(); // inject STATE_SYNC message to emulate start of a new sequence - helper.recordPrivateApi('call.channel.processMessage'); - helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); - await channel.processMessage( - createPM({ - action: 20, - channel: 'channel', - // have cursor so client awaits for additional STATE_SYNC messages - channelSerial: 'serial:cursor', - state: [], - }), - ); + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + // have cursor so client awaits for additional STATE_SYNC messages + syncSerial: 'serial:cursor', + }); let getRootResolved = false; - let newRoot; + let root; liveObjects.getRoot().then((value) => { getRootResolved = true; - newRoot = value; + root = value; }); // wait for next tick to check that getRoot() promise handler didn't proc @@ -297,42 +269,26 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(getRootResolved, 'Check getRoot() is not resolved while STATE_SYNC is in progress').to.be.false; - // inject next STATE_SYNC message - helper.recordPrivateApi('call.channel.processMessage'); - helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); - await channel.processMessage( - createPM({ - action: 20, - channel: 'channel', - // no cursor to indicate the end of STATE_SYNC messages - channelSerial: 'serial:', - state: [ - { - object: { - objectId: 'root', - regionalTimeserial: 'a@0-0', - map: { - entries: { - key: { - timeserial: 'a@0-0', - data: { - value: 1, - }, - }, - }, - }, - }, - }, - ], - }), - ); + // inject final STATE_SYNC message + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + // no cursor to indicate the end of STATE_SYNC messages + syncSerial: 'serial:', + state: [ + liveObjectsHelper.mapObject({ + objectId: 'root', + regionalTimeserial: '@0-0', + entries: { key: { timeserial: '@0-0', data: { value: 1 } } }, + }), + ], + }); // wait for next tick for getRoot() handler to process helper.recordPrivateApi('call.Platform.nextTick'); await new Promise((res) => nextTick(res)); expect(getRootResolved, 'Check getRoot() is resolved when STATE_SYNC sequence has ended').to.be.true; - expect(newRoot.get('key')).to.equal(1, 'Check new root after STATE_SYNC sequence has expected key'); + expect(root.get('key')).to.equal(1, 'Check new root after STATE_SYNC sequence has expected key'); }, client); }); From d919f8b1f81dbfac25bd9993070b5b320e30b112 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 24 Oct 2024 23:28:45 +0100 Subject: [PATCH 055/166] Add LiveObjects tests for buffering and flushing operations outside of sync sequence --- test/realtime/live_objects.test.js | 327 ++++++++++++++++++++++++++++- 1 file changed, 326 insertions(+), 1 deletion(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 0402e39dac..4d472d1a47 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -236,7 +236,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); /** @nospec */ - it('getRoot() waits for subsequent STATE_SYNC to finish before resolving', async function () { + it('getRoot() waits for STATE_SYNC with empty cursor before resolving', async function () { const helper = this.test.helper; const liveObjectsHelper = new LiveObjectsHelper(helper); const client = RealtimeWithLiveObjects(helper); @@ -459,6 +459,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], data: { value: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9', encoding: 'base64' }, }, { key: 'emptyBytesKey', data: { value: '', encoding: 'base64' } }, + { key: 'maxSafeIntegerKey', data: { value: Number.MAX_SAFE_INTEGER } }, + { key: 'negativeMaxSafeIntegerKey', data: { value: -Number.MAX_SAFE_INTEGER } }, { key: 'numberKey', data: { value: 1 } }, { key: 'zeroKey', data: { value: 0 } }, { key: 'trueKey', data: { value: true } }, @@ -902,6 +904,329 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, client); }); } + + const operationsDuringSyncSequence = [ + { + description: 'state operation messages are buffered during STATE_SYNC sequence', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:cursor', + }); + + // inject operations, it should not be applied as sync is in progress + await Promise.all( + primitiveKeyData.map((keyData) => + liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: '@0-0', + state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], + }), + ), + ); + + // check root doesn't have data from operations + primitiveKeyData.forEach((keyData) => { + expect(root.get(keyData.key), `Check "${keyData.key}" key doesn't exist on root during STATE_SYNC`).to.not + .exist; + }); + }, + }, + + { + description: 'buffered state operation messages are applied when STATE_SYNC sequence ends', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:cursor', + }); + + // inject operations, they should be applied when sync ends + await Promise.all( + primitiveKeyData.map((keyData) => + liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: '@0-0', + state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], + }), + ), + ); + + // end the sync with empty cursor + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:', + }); + + // check everything is applied correctly + primitiveKeyData.forEach((keyData) => { + if (keyData.data.encoding) { + expect( + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), + `Check root has correct value for "${keyData.key}" key after STATE_SYNC has ended and buffered operations are applied`, + ).to.be.true; + } else { + expect(root.get(keyData.key)).to.equal( + keyData.data.value, + `Check root has correct value for "${keyData.key}" key after STATE_SYNC has ended and buffered operations are applied`, + ); + } + }); + }, + }, + + { + description: 'buffered state operation messages are discarded when new STATE_SYNC sequence starts', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:cursor', + }); + + // inject operations, expect them to be discarded when sync with new sequence id starts + await Promise.all( + primitiveKeyData.map((keyData) => + liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: '@0-0', + state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], + }), + ), + ); + + // start new sync with new sequence id + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'otherserial:cursor', + }); + + // inject another operation that should be applied when latest sync ends + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: '@0-0', + state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { value: 'bar' } })], + }); + + // end sync + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'otherserial:', + }); + + // check root doesn't have data from operations received during first sync + primitiveKeyData.forEach((keyData) => { + expect( + root.get(keyData.key), + `Check "${keyData.key}" key doesn't exist on root when STATE_SYNC has ended`, + ).to.not.exist; + }); + + // check root has data from operations received during second sync + expect(root.get('foo')).to.equal( + 'bar', + 'Check root has data from operations received during second STATE_SYNC sequence', + ); + }, + }, + + { + description: 'buffered state operation messages are applied based on regional timeserial of the object', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages + const mapId = liveObjectsHelper.fakeMapObjectId(); + const counterId = liveObjectsHelper.fakeCounterObjectId(); + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:cursor', + // add state object messages with non-zero regional timeserials + state: [ + liveObjectsHelper.mapObject({ + objectId: 'root', + regionalTimeserial: '@1-0', + entries: { + map: { timeserial: '@0-0', data: { objectId: mapId } }, + counter: { timeserial: '@0-0', data: { objectId: counterId } }, + }, + }), + liveObjectsHelper.mapObject({ + objectId: mapId, + regionalTimeserial: '@1-0', + }), + liveObjectsHelper.counterObject({ + objectId: counterId, + regionalTimeserial: '@1-0', + }), + ], + }); + + // inject operations with older regional timeserial, expect them not to be applied when sync ends + await Promise.all( + ['root', mapId].flatMap((objectId) => + primitiveKeyData.map((keyData) => + liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: '@0-0', + state: [liveObjectsHelper.mapSetOp({ objectId, key: keyData.key, data: keyData.data })], + }), + ), + ), + ); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: '@0-0', + state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], + }); + + // inject operations with greater regional timeserial, expect them to be applied when sync ends + await Promise.all( + ['root', mapId].map((objectId) => + liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: '@2-0', + state: [liveObjectsHelper.mapSetOp({ objectId, key: 'foo', data: { value: 'bar' } })], + }), + ), + ); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: '@2-0', + state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], + }); + + // end sync + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:', + }); + + // check operations with older regional timeserial are not applied + // counter will be checked to match an expected value explicitly, so no need to check that it doesn't equal a sum of operations + primitiveKeyData.forEach((keyData) => { + expect( + root.get(keyData.key), + `Check "${keyData.key}" key doesn't exist on root when STATE_SYNC has ended`, + ).to.not.exist; + }); + primitiveKeyData.forEach((keyData) => { + expect( + root.get('map').get(keyData.key), + `Check "${keyData.key}" key doesn't exist on inner map when STATE_SYNC has ended`, + ).to.not.exist; + }); + + // check operations with greater regional timeserial are applied + expect(root.get('foo')).to.equal( + 'bar', + 'Check only data from operations with greater regional timeserial exists on root after STATE_SYNC', + ); + expect(root.get('map').get('foo')).to.equal( + 'bar', + 'Check only data from operations with greater regional timeserial exists on inner map after STATE_SYNC', + ); + expect(root.get('counter').value()).to.equal( + 1, + 'Check only increment operations with greater regional timeserial were applied to counter after STATE_SYNC', + ); + }, + }, + + { + description: + 'subsequent state operation messages are applied immediately after STATE_SYNC ended and buffers are applied', + action: async (ctx) => { + const { root, liveObjectsHelper, channel, channelName } = ctx; + + // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:cursor', + }); + + // inject operations, they should be applied when sync ends + await Promise.all( + primitiveKeyData.map((keyData) => + liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: '@0-0', + state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], + }), + ), + ); + + // end the sync with empty cursor + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:', + }); + + // send some more operations + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapSetOp({ + objectId: 'root', + key: 'foo', + data: { value: 'bar' }, + }), + ); + + // check buffered operations are applied, as well as the most recent operation outside of the STATE_SYNC is applied + primitiveKeyData.forEach((keyData) => { + if (keyData.data.encoding) { + expect( + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), + `Check root has correct value for "${keyData.key}" key after STATE_SYNC has ended and buffered operations are applied`, + ).to.be.true; + } else { + expect(root.get(keyData.key)).to.equal( + keyData.data.value, + `Check root has correct value for "${keyData.key}" key after STATE_SYNC has ended and buffered operations are applied`, + ); + } + }); + expect(root.get('foo')).to.equal( + 'bar', + 'Check root has correct value for "foo" key from operation received outside of STATE_SYNC after other buffered operations were applied', + ); + }, + }, + ]; + + for (const scenario of operationsDuringSyncSequence) { + if (scenario.skip === true) { + continue; + } + + /** @nospec */ + it(scenario.description, async function () { + const helper = this.test.helper; + const liveObjectsHelper = new LiveObjectsHelper(helper); + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channelName = scenario.description; + const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + // wait for getRoot() to resolve so the initial SYNC sequence is completed, + // as we're going to initiate a new one to test applying operations during SYNC sequence. + const root = await liveObjects.getRoot(); + + await scenario.action({ root, liveObjectsHelper, channelName, channel }); + }, client); + }); + } }); /** @nospec */ From 868560ca245111cdd0099a168cfb4f0753a225c3 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 25 Oct 2024 11:52:55 +0100 Subject: [PATCH 056/166] Apply timeserial dependant operations only if op timeserial is > than object/entry timeserial --- src/plugins/liveobjects/livemap.ts | 18 +++++++---- src/plugins/liveobjects/liveobjectspool.ts | 9 ++++-- test/realtime/live_objects.test.js | 36 ++++++++++++---------- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 42d8248e7d..b0427a9a07 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -224,13 +224,16 @@ export class LiveMap extends LiveObject { const { ErrorInfo, Utils } = this._client; const existingEntry = this._dataRef.data.get(op.key); - if (existingEntry && opOriginTimeserial.before(existingEntry.timeserial)) { - // the operation's origin timeserial < the entry's timeserial, ignore the operation. + if ( + existingEntry && + (opOriginTimeserial.before(existingEntry.timeserial) || opOriginTimeserial.equal(existingEntry.timeserial)) + ) { + // the operation's origin timeserial <= the entry's timeserial, ignore the operation. this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MICRO, 'LiveMap._applyMapSet()', - `skipping updating key="${op.key}" as existing key entry has greater timeserial: ${existingEntry.timeserial.toString()}, than the op: ${opOriginTimeserial.toString()}; objectId=${this._objectId}`, + `skipping update for key="${op.key}": op timeserial ${opOriginTimeserial.toString()} <= entry timeserial ${existingEntry.timeserial.toString()}; objectId=${this._objectId}`, ); return; } @@ -271,13 +274,16 @@ export class LiveMap extends LiveObject { private _applyMapRemove(op: StateMapOp, opOriginTimeserial: Timeserial): void { const existingEntry = this._dataRef.data.get(op.key); - if (existingEntry && opOriginTimeserial.before(existingEntry.timeserial)) { - // the operation's origin timeserial < the entry's timeserial, ignore the operation. + if ( + existingEntry && + (opOriginTimeserial.before(existingEntry.timeserial) || opOriginTimeserial.equal(existingEntry.timeserial)) + ) { + // the operation's origin timeserial <= the entry's timeserial, ignore the operation. this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MICRO, 'LiveMap._applyMapRemove()', - `skipping removing key="${op.key}" as existing key entry has greater timeserial: ${existingEntry.timeserial.toString()}, than the op: ${opOriginTimeserial.toString()}; objectId=${this._objectId}`, + `skipping remove for key="${op.key}": op timeserial ${opOriginTimeserial.toString()} <= entry timeserial ${existingEntry.timeserial.toString()}; objectId=${this._objectId}`, ); return; } diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index 3d41e2e238..aef14cc8d0 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -151,13 +151,16 @@ export class LiveObjectsPool { } // otherwise we need to compare regional timeserials - if (regionalTimeserial.before(existingObject.getRegionalTimeserial())) { - // the operation's regional timeserial < the object's timeserial, ignore the operation. + if ( + regionalTimeserial.before(existingObject.getRegionalTimeserial()) || + regionalTimeserial.equal(existingObject.getRegionalTimeserial()) + ) { + // the operation's regional timeserial <= the object's timeserial, ignore the operation. this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MICRO, 'LiveObjects.LiveObjectsPool.applyBufferedStateMessages()', - `skipping applying buffered state operation message as existing object has greater regional timeserial: ${existingObject.getRegionalTimeserial().toString()}, than the op: ${regionalTimeserial.toString()}; objectId=${stateMessage.operation.objectId}, message id: ${stateMessage.id}, channel: ${this._channel.name}`, + `skipping buffered state operation message: op regional timeserial ${regionalTimeserial.toString()} <= object regional timeserial ${existingObject.getRegionalTimeserial().toString()}; objectId=${stateMessage.operation.objectId}, message id: ${stateMessage.id}, channel: ${this._channel.name}`, ); continue; } diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 4d472d1a47..a1a9ffaefa 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -1070,23 +1070,27 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ], }); - // inject operations with older regional timeserial, expect them not to be applied when sync ends + // inject operations with older or equal regional timeserial, expect them not to be applied when sync ends await Promise.all( - ['root', mapId].flatMap((objectId) => - primitiveKeyData.map((keyData) => - liveObjectsHelper.processStateOperationMessageOnChannel({ - channel, - serial: '@0-0', - state: [liveObjectsHelper.mapSetOp({ objectId, key: keyData.key, data: keyData.data })], - }), - ), - ), + ['@0-0', '@1-0'].map(async (serial) => { + await Promise.all( + ['root', mapId].flatMap((objectId) => + primitiveKeyData.map((keyData) => + liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial, + state: [liveObjectsHelper.mapSetOp({ objectId, key: keyData.key, data: keyData.data })], + }), + ), + ), + ); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial, + state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], + }); + }), ); - await liveObjectsHelper.processStateOperationMessageOnChannel({ - channel, - serial: '@0-0', - state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], - }); // inject operations with greater regional timeserial, expect them to be applied when sync ends await Promise.all( @@ -1110,7 +1114,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], syncSerial: 'serial:', }); - // check operations with older regional timeserial are not applied + // check operations with older or equal regional timeserial are not applied // counter will be checked to match an expected value explicitly, so no need to check that it doesn't equal a sum of operations primitiveKeyData.forEach((keyData) => { expect( From 30d4c9ccc20b638a964e05facb2e8c0113fcc726 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 5 Nov 2024 04:04:14 +0000 Subject: [PATCH 057/166] Add `deep-equal` package Also configure LiveObjects plugin default bundle to not include `deep-equal` package in the output. --- grunt/esbuild/build.js | 1 + package-lock.json | 144 +++++++++-------------------------------- package.json | 2 + 3 files changed, 33 insertions(+), 114 deletions(-) diff --git a/grunt/esbuild/build.js b/grunt/esbuild/build.js index 728d48b79a..915f7f7955 100644 --- a/grunt/esbuild/build.js +++ b/grunt/esbuild/build.js @@ -82,6 +82,7 @@ const liveObjectsPluginConfig = { entryPoints: ['src/plugins/liveobjects/index.ts'], plugins: [umdWrapper.default({ libraryName: 'AblyLiveObjectsPlugin', amdNamedModule: false })], outfile: 'build/liveobjects.js', + external: ['deep-equal'], }; const liveObjectsPluginCdnConfig = { diff --git a/package-lock.json b/package-lock.json index 63325af508..e6fa48ce92 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@ably/msgpack-js": "^0.4.0", + "deep-equal": "^2.2.3", "fastestsmallesttextencoderdecoder": "^1.0.22", "got": "^11.8.5", "ulid": "^2.3.0", @@ -23,6 +24,7 @@ "@babel/traverse": "^7.23.7", "@testing-library/react": "^13.3.0", "@types/cli-table": "^0.3.4", + "@types/deep-equal": "^1.0.4", "@types/jmespath": "^0.15.2", "@types/node": "^18.0.0", "@types/request": "^2.48.7", @@ -1557,6 +1559,12 @@ "integrity": "sha512-GsALrTL69mlwbAw/MHF1IPTadSLZQnsxe7a80G8l4inN/iEXCOcVeT/S7aRc6hbhqzL9qZ314kHPDQnQ3ev+HA==", "dev": true }, + "node_modules/@types/deep-equal": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/deep-equal/-/deep-equal-1.0.4.tgz", + "integrity": "sha512-tqdiS4otQP4KmY0PR3u6KbZ5EWvhNdUoS/jc93UuK23C220lOZ/9TvjfxdPcKvqwwDVtmtSCrnr0p/2dirAxkA==", + "dev": true + }, "node_modules/@types/eslint": { "version": "8.56.2", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.2.tgz", @@ -2320,7 +2328,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz", "integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "is-array-buffer": "^3.0.1" @@ -2513,7 +2520,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -2749,7 +2755,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", - "dev": true, "dependencies": { "function-bind": "^1.1.2", "get-intrinsic": "^1.2.1", @@ -3288,7 +3293,6 @@ "version": "2.2.3", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.3.tgz", "integrity": "sha512-ZIwpnevOurS8bpT4192sqAowWM76JDKSHYzMLty3BZGSswgq6pBaH3DhCSW5xVAZICZyKdOBPjwww5wfgT/6PA==", - "dev": true, "dependencies": { "array-buffer-byte-length": "^1.0.0", "call-bind": "^1.0.5", @@ -3319,8 +3323,7 @@ "node_modules/deep-equal/node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", - "dev": true + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" }, "node_modules/deep-for-each": { "version": "3.0.0", @@ -3349,7 +3352,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", - "dev": true, "dependencies": { "get-intrinsic": "^1.2.1", "gopd": "^1.0.1", @@ -3363,7 +3365,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "dev": true, "dependencies": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", @@ -3663,7 +3664,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz", "integrity": "sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "get-intrinsic": "^1.1.3", @@ -3682,8 +3682,7 @@ "node_modules/es-get-iterator/node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", - "dev": true + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" }, "node_modules/es-iterator-helpers": { "version": "1.0.15", @@ -5013,7 +5012,6 @@ "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dev": true, "dependencies": { "is-callable": "^1.1.3" } @@ -5111,7 +5109,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -5144,7 +5141,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -5180,7 +5176,6 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", - "dev": true, "dependencies": { "function-bind": "^1.1.2", "has-proto": "^1.0.1", @@ -5385,7 +5380,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dev": true, "dependencies": { "get-intrinsic": "^1.1.3" }, @@ -5792,7 +5786,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -5810,7 +5803,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz", "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==", - "dev": true, "dependencies": { "get-intrinsic": "^1.2.2" }, @@ -5822,7 +5814,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -5834,7 +5825,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -5846,7 +5836,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "dev": true, "dependencies": { "has-symbols": "^1.0.2" }, @@ -5861,7 +5850,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", - "dev": true, "dependencies": { "function-bind": "^1.1.2" }, @@ -6077,7 +6065,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.6.tgz", "integrity": "sha512-Xj6dv+PsbtwyPpEflsejS+oIZxmMlV44zAhG479uYu89MsjcYOhCFnNyKrkJrihbsiasQyY0afoCl/9BLR65bg==", - "dev": true, "dependencies": { "get-intrinsic": "^1.2.2", "hasown": "^2.0.0", @@ -6122,7 +6109,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -6138,7 +6124,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz", "integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "get-intrinsic": "^1.2.0", @@ -6167,7 +6152,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", - "dev": true, "dependencies": { "has-bigints": "^1.0.1" }, @@ -6191,7 +6175,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -6213,7 +6196,6 @@ "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -6237,7 +6219,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", - "dev": true, "dependencies": { "has-tostringtag": "^1.0.0" }, @@ -6324,7 +6305,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.2.tgz", "integrity": "sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -6354,7 +6334,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", - "dev": true, "dependencies": { "has-tostringtag": "^1.0.0" }, @@ -6396,7 +6375,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -6424,7 +6402,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.2.tgz", "integrity": "sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -6433,7 +6410,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", - "dev": true, "dependencies": { "call-bind": "^1.0.2" }, @@ -6445,7 +6421,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", - "dev": true, "dependencies": { "has-tostringtag": "^1.0.0" }, @@ -6460,7 +6435,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", - "dev": true, "dependencies": { "has-symbols": "^1.0.2" }, @@ -6475,7 +6449,6 @@ "version": "1.1.12", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", - "dev": true, "dependencies": { "which-typed-array": "^1.1.11" }, @@ -6502,7 +6475,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz", "integrity": "sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -6523,7 +6495,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.2.tgz", "integrity": "sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "get-intrinsic": "^1.1.1" @@ -7590,7 +7561,6 @@ "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -7599,7 +7569,6 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz", "integrity": "sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.3" @@ -7615,7 +7584,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, "engines": { "node": ">= 0.4" } @@ -7624,7 +7592,6 @@ "version": "4.1.5", "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz", "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==", - "dev": true, "dependencies": { "call-bind": "^1.0.5", "define-properties": "^1.2.1", @@ -8565,7 +8532,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz", "integrity": "sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.2.0", @@ -8995,7 +8961,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.0.tgz", "integrity": "sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w==", - "dev": true, "dependencies": { "define-data-property": "^1.1.1", "function-bind": "^1.1.2", @@ -9011,7 +8976,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.1.tgz", "integrity": "sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==", - "dev": true, "dependencies": { "define-data-property": "^1.0.1", "functions-have-names": "^1.2.3", @@ -9113,7 +9077,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", - "dev": true, "dependencies": { "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", @@ -9304,7 +9267,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz", "integrity": "sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==", - "dev": true, "dependencies": { "internal-slot": "^1.0.4" }, @@ -10865,7 +10827,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", - "dev": true, "dependencies": { "is-bigint": "^1.0.1", "is-boolean-object": "^1.1.0", @@ -10913,7 +10874,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.1.tgz", "integrity": "sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==", - "dev": true, "dependencies": { "is-map": "^2.0.1", "is-set": "^2.0.1", @@ -10928,7 +10888,6 @@ "version": "1.1.13", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.13.tgz", "integrity": "sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==", - "dev": true, "dependencies": { "available-typed-arrays": "^1.0.5", "call-bind": "^1.0.4", @@ -12171,6 +12130,12 @@ "integrity": "sha512-GsALrTL69mlwbAw/MHF1IPTadSLZQnsxe7a80G8l4inN/iEXCOcVeT/S7aRc6hbhqzL9qZ314kHPDQnQ3ev+HA==", "dev": true }, + "@types/deep-equal": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/deep-equal/-/deep-equal-1.0.4.tgz", + "integrity": "sha512-tqdiS4otQP4KmY0PR3u6KbZ5EWvhNdUoS/jc93UuK23C220lOZ/9TvjfxdPcKvqwwDVtmtSCrnr0p/2dirAxkA==", + "dev": true + }, "@types/eslint": { "version": "8.56.2", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.2.tgz", @@ -12770,7 +12735,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz", "integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==", - "dev": true, "requires": { "call-bind": "^1.0.2", "is-array-buffer": "^3.0.1" @@ -12913,8 +12877,7 @@ "available-typed-arrays": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", - "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", - "dev": true + "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" }, "aws-sdk": { "version": "2.1539.0", @@ -13095,7 +13058,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", - "dev": true, "requires": { "function-bind": "^1.1.2", "get-intrinsic": "^1.2.1", @@ -13494,7 +13456,6 @@ "version": "2.2.3", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.3.tgz", "integrity": "sha512-ZIwpnevOurS8bpT4192sqAowWM76JDKSHYzMLty3BZGSswgq6pBaH3DhCSW5xVAZICZyKdOBPjwww5wfgT/6PA==", - "dev": true, "requires": { "array-buffer-byte-length": "^1.0.0", "call-bind": "^1.0.5", @@ -13519,8 +13480,7 @@ "isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", - "dev": true + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" } } }, @@ -13548,7 +13508,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", - "dev": true, "requires": { "get-intrinsic": "^1.2.1", "gopd": "^1.0.1", @@ -13559,7 +13518,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "dev": true, "requires": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", @@ -13790,7 +13748,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz", "integrity": "sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw==", - "dev": true, "requires": { "call-bind": "^1.0.2", "get-intrinsic": "^1.1.3", @@ -13806,8 +13763,7 @@ "isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", - "dev": true + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" } } }, @@ -14765,7 +14721,6 @@ "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dev": true, "requires": { "is-callable": "^1.1.3" } @@ -14834,8 +14789,7 @@ "function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" }, "function.prototype.name": { "version": "1.1.6", @@ -14858,8 +14812,7 @@ "functions-have-names": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", - "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", - "dev": true + "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==" }, "gensync": { "version": "1.0.0-beta.2", @@ -14883,7 +14836,6 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", - "dev": true, "requires": { "function-bind": "^1.1.2", "has-proto": "^1.0.1", @@ -15032,7 +14984,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dev": true, "requires": { "get-intrinsic": "^1.1.3" } @@ -15346,8 +15297,7 @@ "has-bigints": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", - "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", - "dev": true + "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==" }, "has-flag": { "version": "4.0.0", @@ -15359,7 +15309,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz", "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==", - "dev": true, "requires": { "get-intrinsic": "^1.2.2" } @@ -15367,20 +15316,17 @@ "has-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", - "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", - "dev": true + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==" }, "has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "dev": true + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" }, "has-tostringtag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "dev": true, "requires": { "has-symbols": "^1.0.2" } @@ -15389,7 +15335,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", - "dev": true, "requires": { "function-bind": "^1.1.2" } @@ -15551,7 +15496,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.6.tgz", "integrity": "sha512-Xj6dv+PsbtwyPpEflsejS+oIZxmMlV44zAhG479uYu89MsjcYOhCFnNyKrkJrihbsiasQyY0afoCl/9BLR65bg==", - "dev": true, "requires": { "get-intrinsic": "^1.2.2", "hasown": "^2.0.0", @@ -15584,7 +15528,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "dev": true, "requires": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -15594,7 +15537,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz", "integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==", - "dev": true, "requires": { "call-bind": "^1.0.2", "get-intrinsic": "^1.2.0", @@ -15614,7 +15556,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", - "dev": true, "requires": { "has-bigints": "^1.0.1" } @@ -15632,7 +15573,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", - "dev": true, "requires": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -15647,8 +15587,7 @@ "is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "dev": true + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==" }, "is-core-module": { "version": "2.13.1", @@ -15663,7 +15602,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", - "dev": true, "requires": { "has-tostringtag": "^1.0.0" } @@ -15716,8 +15654,7 @@ "is-map": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.2.tgz", - "integrity": "sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==", - "dev": true + "integrity": "sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==" }, "is-negative-zero": { "version": "2.0.2", @@ -15735,7 +15672,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", - "dev": true, "requires": { "has-tostringtag": "^1.0.0" } @@ -15765,7 +15701,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", - "dev": true, "requires": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -15783,14 +15718,12 @@ "is-set": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.2.tgz", - "integrity": "sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==", - "dev": true + "integrity": "sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==" }, "is-shared-array-buffer": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", - "dev": true, "requires": { "call-bind": "^1.0.2" } @@ -15799,7 +15732,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", - "dev": true, "requires": { "has-tostringtag": "^1.0.0" } @@ -15808,7 +15740,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", - "dev": true, "requires": { "has-symbols": "^1.0.2" } @@ -15817,7 +15748,6 @@ "version": "1.1.12", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", - "dev": true, "requires": { "which-typed-array": "^1.1.11" } @@ -15834,8 +15764,7 @@ "is-weakmap": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz", - "integrity": "sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==", - "dev": true + "integrity": "sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==" }, "is-weakref": { "version": "1.0.2", @@ -15850,7 +15779,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.2.tgz", "integrity": "sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg==", - "dev": true, "requires": { "call-bind": "^1.0.2", "get-intrinsic": "^1.1.1" @@ -16657,14 +16585,12 @@ "object-inspect": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", - "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", - "dev": true + "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==" }, "object-is": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz", "integrity": "sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==", - "dev": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3" @@ -16673,14 +16599,12 @@ "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" }, "object.assign": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz", "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==", - "dev": true, "requires": { "call-bind": "^1.0.5", "define-properties": "^1.2.1", @@ -17349,7 +17273,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz", "integrity": "sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg==", - "dev": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.2.0", @@ -17665,7 +17588,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.0.tgz", "integrity": "sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w==", - "dev": true, "requires": { "define-data-property": "^1.1.1", "function-bind": "^1.1.2", @@ -17678,7 +17600,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.1.tgz", "integrity": "sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==", - "dev": true, "requires": { "define-data-property": "^1.0.1", "functions-have-names": "^1.2.3", @@ -17758,7 +17679,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", - "dev": true, "requires": { "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", @@ -17907,7 +17827,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz", "integrity": "sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==", - "dev": true, "requires": { "internal-slot": "^1.0.4" } @@ -18978,7 +18897,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", - "dev": true, "requires": { "is-bigint": "^1.0.1", "is-boolean-object": "^1.1.0", @@ -19019,7 +18937,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.1.tgz", "integrity": "sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==", - "dev": true, "requires": { "is-map": "^2.0.1", "is-set": "^2.0.1", @@ -19031,7 +18948,6 @@ "version": "1.1.13", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.13.tgz", "integrity": "sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==", - "dev": true, "requires": { "available-typed-arrays": "^1.0.5", "call-bind": "^1.0.4", diff --git a/package.json b/package.json index 3cad5645ef..b8f9ae029e 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ ], "dependencies": { "@ably/msgpack-js": "^0.4.0", + "deep-equal": "^2.2.3", "fastestsmallesttextencoderdecoder": "^1.0.22", "got": "^11.8.5", "ulid": "^2.3.0", @@ -72,6 +73,7 @@ "@babel/traverse": "^7.23.7", "@testing-library/react": "^13.3.0", "@types/cli-table": "^0.3.4", + "@types/deep-equal": "^1.0.4", "@types/jmespath": "^0.15.2", "@types/node": "^18.0.0", "@types/request": "^2.48.7", From b40dc9f7a8dfe393723bc8f77cc01522fbc1903d Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 5 Nov 2024 04:25:35 +0000 Subject: [PATCH 058/166] Implement direct object subscription for Live Objects - subscription callback is invoked when the live object data is updated via incoming state operation - subscription callback is invoked when the live object data is updated via a sync sequence (once sync sequence is applied to all objects) - update object is passed to a callback function that describes a granular update made to the live object Resolves DTP-958 --- src/plugins/liveobjects/livecounter.ts | 32 ++++-- src/plugins/liveobjects/livemap.ts | 116 ++++++++++++++++++--- src/plugins/liveobjects/liveobject.ts | 75 ++++++++++++- src/plugins/liveobjects/liveobjects.ts | 11 +- src/plugins/liveobjects/liveobjectspool.ts | 2 - 5 files changed, 203 insertions(+), 33 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 61d17d9a24..5ec413434b 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -1,4 +1,4 @@ -import { LiveObject, LiveObjectData } from './liveobject'; +import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; import { StateCounter, StateCounterOp, StateMessage, StateOperation, StateOperationAction } from './statemessage'; import { Timeserial } from './timeserial'; @@ -7,7 +7,11 @@ export interface LiveCounterData extends LiveObjectData { data: number; } -export class LiveCounter extends LiveObject { +export interface LiveCounterUpdate extends LiveObjectUpdate { + update: { inc: number }; +} + +export class LiveCounter extends LiveObject { constructor( liveObjects: LiveObjects, private _created: boolean, @@ -62,16 +66,19 @@ export class LiveCounter extends LiveObject { ); } + let update: LiveCounterUpdate | LiveObjectUpdateNoop; switch (op.action) { case StateOperationAction.COUNTER_CREATE: - this._applyCounterCreate(op.counter); + update = this._applyCounterCreate(op.counter); break; case StateOperationAction.COUNTER_INC: if (this._client.Utils.isNil(op.counterOp)) { this._throwNoPayloadError(op); + // leave an explicit return here, so that TS knows that update object is always set after the switch statement. + return; } else { - this._applyCounterInc(op.counterOp); + update = this._applyCounterInc(op.counterOp); } break; @@ -84,12 +91,18 @@ export class LiveCounter extends LiveObject { } this.setRegionalTimeserial(opRegionalTimeserial); + this.notifyUpdated(update); } protected _getZeroValueData(): LiveCounterData { return { data: 0 }; } + protected _updateFromDataDiff(currentDataRef: LiveCounterData, newDataRef: LiveCounterData): LiveCounterUpdate { + const counterDiff = newDataRef.data - currentDataRef.data; + return { update: { inc: counterDiff } }; + } + private _throwNoPayloadError(op: StateOperation): void { throw new this._client.ErrorInfo( `No payload found for ${op.action} op for LiveCounter objectId=${this.getObjectId()}`, @@ -98,7 +111,7 @@ export class LiveCounter extends LiveObject { ); } - private _applyCounterCreate(op: StateCounter | undefined): void { + private _applyCounterCreate(op: StateCounter | undefined): LiveCounterUpdate | LiveObjectUpdateNoop { if (this.isCreated()) { // skip COUNTER_CREATE op if this counter is already created this._client.Logger.logAction( @@ -107,14 +120,14 @@ export class LiveCounter extends LiveObject { 'LiveCounter._applyCounterCreate()', `skipping applying COUNTER_CREATE op on a counter instance as it is already created; objectId=${this._objectId}`, ); - return; + return { noop: true }; } if (this._client.Utils.isNil(op)) { // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly 0 in this case. // we need to SUM the initial value to the current value due to the reasons below, but since it's a 0, we can skip addition operation this.setCreated(true); - return; + return { update: { inc: 0 } }; } // note that it is intentional to SUM the incoming count from the create op. @@ -122,9 +135,12 @@ export class LiveCounter extends LiveObject { // so it is missing the initial value that we're going to add now. this._dataRef.data += op.count ?? 0; this.setCreated(true); + + return { update: { inc: op.count ?? 0 } }; } - private _applyCounterInc(op: StateCounterOp): void { + private _applyCounterInc(op: StateCounterOp): LiveCounterUpdate { this._dataRef.data += op.amount; + return { update: { inc: op.amount } }; } } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index b0427a9a07..97129f8964 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -1,5 +1,7 @@ +import deepEqual from 'deep-equal'; + import type BaseClient from 'common/lib/client/baseclient'; -import { LiveObject, LiveObjectData } from './liveobject'; +import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; import { MapSemantics, @@ -40,7 +42,11 @@ export interface LiveMapData extends LiveObjectData { data: Map; } -export class LiveMap extends LiveObject { +export interface LiveMapUpdate extends LiveObjectUpdate { + update: { [keyName: string]: 'updated' | 'removed' }; +} + +export class LiveMap extends LiveObject { constructor( liveObjects: LiveObjects, private _semantics: MapSemantics, @@ -60,6 +66,9 @@ export class LiveMap extends LiveObject { return new LiveMap(liveobjects, MapSemantics.LWW, null, objectId, regionalTimeserial); } + /** + * @internal + */ static liveMapDataFromMapEntries(client: BaseClient, entries: Record): LiveMapData { const liveMapData: LiveMapData = { data: new Map(), @@ -122,7 +131,7 @@ export class LiveMap extends LiveObject { let size = 0; for (const value of this._dataRef.data.values()) { if (value.tombstone === true) { - // should not count deleted entries + // should not count removed entries continue; } @@ -144,24 +153,29 @@ export class LiveMap extends LiveObject { ); } + let update: LiveMapUpdate | LiveObjectUpdateNoop; switch (op.action) { case StateOperationAction.MAP_CREATE: - this._applyMapCreate(op.map); + update = this._applyMapCreate(op.map); break; case StateOperationAction.MAP_SET: if (this._client.Utils.isNil(op.mapOp)) { this._throwNoPayloadError(op); + // leave an explicit return here, so that TS knows that update object is always set after the switch statement. + return; } else { - this._applyMapSet(op.mapOp, DefaultTimeserial.calculateTimeserial(this._client, msg.serial)); + update = this._applyMapSet(op.mapOp, DefaultTimeserial.calculateTimeserial(this._client, msg.serial)); } break; case StateOperationAction.MAP_REMOVE: if (this._client.Utils.isNil(op.mapOp)) { this._throwNoPayloadError(op); + // leave an explicit return here, so that TS knows that update object is always set after the switch statement. + return; } else { - this._applyMapRemove(op.mapOp, DefaultTimeserial.calculateTimeserial(this._client, msg.serial)); + update = this._applyMapRemove(op.mapOp, DefaultTimeserial.calculateTimeserial(this._client, msg.serial)); } break; @@ -174,12 +188,67 @@ export class LiveMap extends LiveObject { } this.setRegionalTimeserial(opRegionalTimeserial); + this.notifyUpdated(update); } protected _getZeroValueData(): LiveMapData { return { data: new Map() }; } + protected _updateFromDataDiff(currentDataRef: LiveMapData, newDataRef: LiveMapData): LiveMapUpdate { + const update: LiveMapUpdate = { update: {} }; + + for (const [key, currentEntry] of currentDataRef.data.entries()) { + // any non-tombstoned properties that exist on a current map, but not in the new data - got removed + if (currentEntry.tombstone === false && !newDataRef.data.has(key)) { + update.update[key] = 'removed'; + } + } + + for (const [key, newEntry] of newDataRef.data.entries()) { + if (!currentDataRef.data.has(key)) { + // if property does not exist in the current map, but new data has it as a non-tombstoned property - got updated + if (newEntry.tombstone === false) { + update.update[key] = 'updated'; + continue; + } + + // otherwise, if new data has this prop tombstoned - do nothing, as property didn't exist anyway + if (newEntry.tombstone === true) { + continue; + } + } + + // properties that exist both in current and new map data need to have their values compared to decide on the update type + const currentEntry = currentDataRef.data.get(key)!; + + // compare tombstones first + if (currentEntry.tombstone === true && newEntry.tombstone === false) { + // current prop is tombstoned, but new is not. it means prop was updated to a meaningful value + update.update[key] = 'updated'; + continue; + } + if (currentEntry.tombstone === false && newEntry.tombstone === true) { + // current prop is not tombstoned, but new is. it means prop was removed + update.update[key] = 'removed'; + continue; + } + if (currentEntry.tombstone === true && newEntry.tombstone === true) { + // both props are tombstoned - treat as noop, as there is no data to compare. + continue; + } + + // both props exist and are not tombstoned, need to compare values with deep equals to see if it was changed + const valueChanged = !deepEqual(currentEntry.data, newEntry.data, { strict: true }); + if (valueChanged) { + update.update[key] = 'updated'; + continue; + } + } + + return update; + } + private _throwNoPayloadError(op: StateOperation): void { throw new this._client.ErrorInfo( `No payload found for ${op.action} op for LiveMap objectId=${this.getObjectId()}`, @@ -188,11 +257,11 @@ export class LiveMap extends LiveObject { ); } - private _applyMapCreate(op: StateMap | undefined): void { + private _applyMapCreate(op: StateMap | undefined): LiveMapUpdate | LiveObjectUpdateNoop { if (this._client.Utils.isNil(op)) { // if a map object is missing for the MAP_CREATE op, the initial value is implicitly an empty map. // in this case there is nothing to merge into the current map, so we can just end processing the op. - return; + return { update: {} }; } if (this._semantics !== op.semantics) { @@ -203,6 +272,7 @@ export class LiveMap extends LiveObject { ); } + const aggregatedUpdate: LiveMapUpdate | LiveObjectUpdateNoop = { update: {} }; // in order to apply MAP_CREATE op for an existing map, we should merge their underlying entries keys. // we can do this by iterating over entries from MAP_CREATE op and apply changes on per-key basis as if we had MAP_SET, MAP_REMOVE operations. Object.entries(op.entries ?? {}).forEach(([key, entry]) => { @@ -210,17 +280,28 @@ export class LiveMap extends LiveObject { const opOriginTimeserial = entry.timeserial ? DefaultTimeserial.calculateTimeserial(this._client, entry.timeserial) : DefaultTimeserial.zeroValueTimeserial(this._client); + let update: LiveMapUpdate | LiveObjectUpdateNoop; if (entry.tombstone === true) { - // entry in MAP_CREATE op is deleted, try to apply MAP_REMOVE op - this._applyMapRemove({ key }, opOriginTimeserial); + // entry in MAP_CREATE op is removed, try to apply MAP_REMOVE op + update = this._applyMapRemove({ key }, opOriginTimeserial); } else { - // entry in MAP_CREATE op is not deleted, try to set it via MAP_SET op - this._applyMapSet({ key, data: entry.data }, opOriginTimeserial); + // entry in MAP_CREATE op is not removed, try to set it via MAP_SET op + update = this._applyMapSet({ key, data: entry.data }, opOriginTimeserial); + } + + // skip noop updates + if ((update as LiveObjectUpdateNoop).noop) { + return; } + + // otherwise copy update data to aggregated update + Object.assign(aggregatedUpdate.update, update.update); }); + + return aggregatedUpdate; } - private _applyMapSet(op: StateMapOp, opOriginTimeserial: Timeserial): void { + private _applyMapSet(op: StateMapOp, opOriginTimeserial: Timeserial): LiveMapUpdate | LiveObjectUpdateNoop { const { ErrorInfo, Utils } = this._client; const existingEntry = this._dataRef.data.get(op.key); @@ -235,7 +316,7 @@ export class LiveMap extends LiveObject { 'LiveMap._applyMapSet()', `skipping update for key="${op.key}": op timeserial ${opOriginTimeserial.toString()} <= entry timeserial ${existingEntry.timeserial.toString()}; objectId=${this._objectId}`, ); - return; + return { noop: true }; } if (Utils.isNil(op.data) || (Utils.isNil(op.data.value) && Utils.isNil(op.data.objectId))) { @@ -270,9 +351,10 @@ export class LiveMap extends LiveObject { }; this._dataRef.data.set(op.key, newEntry); } + return { update: { [op.key]: 'updated' } }; } - private _applyMapRemove(op: StateMapOp, opOriginTimeserial: Timeserial): void { + private _applyMapRemove(op: StateMapOp, opOriginTimeserial: Timeserial): LiveMapUpdate | LiveObjectUpdateNoop { const existingEntry = this._dataRef.data.get(op.key); if ( existingEntry && @@ -285,7 +367,7 @@ export class LiveMap extends LiveObject { 'LiveMap._applyMapRemove()', `skipping remove for key="${op.key}": op timeserial ${opOriginTimeserial.toString()} <= entry timeserial ${existingEntry.timeserial.toString()}; objectId=${this._objectId}`, ); - return; + return { noop: true }; } if (existingEntry) { @@ -300,5 +382,7 @@ export class LiveMap extends LiveObject { }; this._dataRef.data.set(op.key, newEntry); } + + return { update: { [op.key]: 'removed' } }; } } diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index 70a294779b..e95eb95d74 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -1,31 +1,76 @@ import type BaseClient from 'common/lib/client/baseclient'; +import type EventEmitter from 'common/lib/util/eventemitter'; import { LiveObjects } from './liveobjects'; import { StateMessage, StateOperation } from './statemessage'; import { DefaultTimeserial, Timeserial } from './timeserial'; +enum LiveObjectEvents { + Updated = 'Updated', +} + export interface LiveObjectData { data: any; } -export abstract class LiveObject { +export interface LiveObjectUpdate { + update: any; +} + +export interface LiveObjectUpdateNoop { + // have optional update field with undefined type so it's not possible to create a noop object with a meaningful update property. + update?: undefined; + noop: true; +} + +export interface SubscribeResponse { + unsubscribe(): void; +} + +export abstract class LiveObject< + TData extends LiveObjectData = LiveObjectData, + TUpdate extends LiveObjectUpdate = LiveObjectUpdate, +> { protected _client: BaseClient; - protected _dataRef: T; + protected _eventEmitter: EventEmitter; + protected _dataRef: TData; protected _objectId: string; protected _regionalTimeserial: Timeserial; constructor( protected _liveObjects: LiveObjects, - initialData?: T | null, + initialData?: TData | null, objectId?: string, regionalTimeserial?: Timeserial, ) { this._client = this._liveObjects.getClient(); + this._eventEmitter = new this._client.EventEmitter(this._client.logger); this._dataRef = initialData ?? this._getZeroValueData(); this._objectId = objectId ?? this._createObjectId(); // use zero value timeserial by default, so any future operation can be applied for this object this._regionalTimeserial = regionalTimeserial ?? DefaultTimeserial.zeroValueTimeserial(this._client); } + subscribe(listener: (update: TUpdate) => void): SubscribeResponse { + this._eventEmitter.on(LiveObjectEvents.Updated, listener); + + const unsubscribe = () => { + this._eventEmitter.off(LiveObjectEvents.Updated, listener); + }; + + return { unsubscribe }; + } + + unsubscribe(listener: (update: TUpdate) => void): void { + // current implementation of the EventEmitter will remove all listeners if .off is called without arguments or with nullish arguments. + // or when called with just an event argument, it will remove all listeners for the event. + // thus we need to check that listener does actually exist before calling .off. + if (this._client.Utils.isNil(listener)) { + return; + } + + this._eventEmitter.off(LiveObjectEvents.Updated, listener); + } + /** * @internal */ @@ -41,10 +86,14 @@ export abstract class LiveObject { } /** + * Sets a new data reference for the LiveObject and returns an update object that describes the changes applied based on the object's previous value. + * * @internal */ - setData(newDataRef: T): void { + setData(newDataRef: TData): TUpdate { + const update = this._updateFromDataDiff(this._dataRef, newDataRef); this._dataRef = newDataRef; + return update; } /** @@ -54,6 +103,18 @@ export abstract class LiveObject { this._regionalTimeserial = regionalTimeserial; } + /** + * @internal + */ + notifyUpdated(update: TUpdate | LiveObjectUpdateNoop): void { + // should not emit update event if update was noop + if ((update as LiveObjectUpdateNoop).noop) { + return; + } + + this._eventEmitter.emit(LiveObjectEvents.Updated, update); + } + private _createObjectId(): string { // TODO: implement object id generation based on live object type and initial value return Math.random().toString().substring(2); @@ -63,5 +124,9 @@ export abstract class LiveObject { * @internal */ abstract applyOperation(op: StateOperation, msg: StateMessage, opRegionalTimeserial: Timeserial): void; - protected abstract _getZeroValueData(): T; + protected abstract _getZeroValueData(): TData; + /** + * Calculate the update object based on the current Live Object data and incoming new data. + */ + protected abstract _updateFromDataDiff(currentDataRef: TData, newDataRef: TData): TUpdate; } diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 9b1981f63a..a5070ec993 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -4,7 +4,7 @@ import type EventEmitter from 'common/lib/util/eventemitter'; import type * as API from '../../../ably'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; -import { LiveObject } from './liveobject'; +import { LiveObject, LiveObjectUpdate } from './liveobject'; import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage } from './statemessage'; import { LiveCounterDataEntry, SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; @@ -199,6 +199,7 @@ export class LiveObjects { } const receivedObjectIds = new Set(); + const existingObjectUpdates: { object: LiveObject; update: LiveObjectUpdate }[] = []; for (const [objectId, entry] of this._syncLiveObjectsDataPool.entries()) { receivedObjectIds.add(objectId); @@ -206,11 +207,14 @@ export class LiveObjects { const regionalTimeserialObj = DefaultTimeserial.calculateTimeserial(this._client, entry.regionalTimeserial); if (existingObject) { - existingObject.setData(entry.objectData); + const update = existingObject.setData(entry.objectData); existingObject.setRegionalTimeserial(regionalTimeserialObj); if (existingObject instanceof LiveCounter) { existingObject.setCreated((entry as LiveCounterDataEntry).created); } + // store updates for existing objects to call subscription callbacks for all of them once the SYNC sequence is completed. + // this will ensure that clients get notified about changes only once everything was applied. + existingObjectUpdates.push({ object: existingObject, update }); continue; } @@ -235,5 +239,8 @@ export class LiveObjects { // need to remove LiveObject instances from the LiveObjectsPool for which objectIds were not received during the SYNC sequence this._liveObjectsPool.deleteExtraObjectIds([...receivedObjectIds]); + + // call subscription callbacks for all updated existing objects + existingObjectUpdates.forEach(({ object, update }) => object.notifyUpdated(update)); } } diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index aef14cc8d0..a4ea8c92ef 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -89,7 +89,6 @@ export class LiveObjectsPool { if (this.get(stateOperation.objectId)) { // object wich such id already exists (we may have created a zero-value object before, or this is a duplicate *_CREATE op), // so delegate application of the op to that object - // TODO: invoke subscription callbacks for an object when applied this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage, regionalTimeserial); break; } @@ -111,7 +110,6 @@ export class LiveObjectsPool { // we create a zero-value object for the provided object id, and apply operation for that zero-value object. // when we eventually receive a corresponding *_CREATE op for that object, its application will be handled by that zero-value object. this.createZeroValueObjectIfNotExists(stateOperation.objectId); - // TODO: invoke subscription callbacks for an object when applied this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage, regionalTimeserial); break; From 07cafc44cbdd1db56a65783f6e76d2b39a55b740 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 5 Nov 2024 04:28:41 +0000 Subject: [PATCH 059/166] Implement .unsubscribeAll() method on Live Objects Resolves DTP-959 --- src/plugins/liveobjects/liveobject.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index e95eb95d74..78244ab650 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -71,6 +71,10 @@ export abstract class LiveObject< this._eventEmitter.off(LiveObjectEvents.Updated, listener); } + unsubscribeAll(): void { + this._eventEmitter.off(LiveObjectEvents.Updated); + } + /** * @internal */ From f8fda9821f73a82bdd14f6e8a556c66a8d653d03 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 5 Nov 2024 06:24:00 +0000 Subject: [PATCH 060/166] Add LiveObjects subscriptions tests Also add `deep-equal` to the list of named dependencies for tests so that browser tests know where to get this package. --- test/common/globals/named_dependencies.js | 1 + test/realtime/live_objects.test.js | 521 +++++++++++++++++++++- 2 files changed, 519 insertions(+), 3 deletions(-) diff --git a/test/common/globals/named_dependencies.js b/test/common/globals/named_dependencies.js index b303f0dfc9..1c26928249 100644 --- a/test/common/globals/named_dependencies.js +++ b/test/common/globals/named_dependencies.js @@ -22,6 +22,7 @@ define(function () { async: { browser: 'node_modules/async/lib/async' }, chai: { browser: 'node_modules/chai/chai', node: 'node_modules/chai/chai' }, ulid: { browser: 'node_modules/ulid/dist/index.umd', node: 'node_modules/ulid/dist/index.umd' }, + 'deep-equal': { browser: 'node_modules/deep-equal/index', node: 'node_modules/deep-equal/index' }, private_api_recorder: { browser: 'test/common/modules/private_api_recorder', node: 'test/common/modules/private_api_recorder', diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index a1a9ffaefa..9353245270 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -562,7 +562,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ), ).to.not.exist; - // create map with references. need to created referenced objects first to obtain their object ids + // create map with references. need to create referenced objects first to obtain their object ids const { objectId: referencedMapObjectId } = await liveObjectsHelper.stateRequest( channelName, liveObjectsHelper.mapCreateOp({ entries: { stringKey: { data: { value: 'stringValue' } } } }), @@ -905,7 +905,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); } - const operationsDuringSyncSequence = [ + const applyOperationsDuringSyncScenarios = [ { description: 'state operation messages are buffered during STATE_SYNC sequence', action: async (ctx) => { @@ -1206,7 +1206,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, ]; - for (const scenario of operationsDuringSyncSequence) { + for (const scenario of applyOperationsDuringSyncScenarios) { if (scenario.skip === true) { continue; } @@ -1231,6 +1231,521 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, client); }); } + + const subscriptionCallbacksScenarios = [ + { + description: 'can subscribe to the incoming COUNTER_INC operation on a LiveCounter', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + + const counter = root.get(sampleCounterKey); + const subscriptionPromise = new Promise((resolve, reject) => + counter.subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { inc: 1 } }, + 'Check counter subscription callback is called with an expected update object for COUNTER_INC operation', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.counterIncOp({ + objectId: sampleCounterObjectId, + amount: 1, + }), + ); + + await subscriptionPromise; + }, + }, + + { + description: 'can subscribe to multiple incoming operations on a LiveCounter', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + + const counter = root.get(sampleCounterKey); + const expectedCounterIncrements = [100, -100, Number.MAX_SAFE_INTEGER, -Number.MAX_SAFE_INTEGER]; + let currentUpdateIndex = 0; + + const subscriptionPromise = new Promise((resolve, reject) => + counter.subscribe((update) => { + try { + const expectedInc = expectedCounterIncrements[currentUpdateIndex]; + expect(update).to.deep.equal( + { update: { inc: expectedInc } }, + `Check counter subscription callback is called with an expected update object for ${currentUpdateIndex + 1} times`, + ); + + if (currentUpdateIndex === expectedCounterIncrements.length - 1) { + resolve(); + } + + currentUpdateIndex++; + } catch (error) { + reject(error); + } + }), + ); + + for (const increment of expectedCounterIncrements) { + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.counterIncOp({ + objectId: sampleCounterObjectId, + amount: increment, + }), + ); + } + + await subscriptionPromise; + }, + }, + + { + description: 'can subscribe to the incoming MAP_SET operation on a LiveMap', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + const subscriptionPromise = new Promise((resolve, reject) => + map.subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { stringKey: 'updated' } }, + 'Check map subscription callback is called with an expected update object for MAP_SET operation', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapSetOp({ + objectId: sampleMapObjectId, + key: 'stringKey', + data: { value: 'stringValue' }, + }), + ); + + await subscriptionPromise; + }, + }, + + { + description: 'can subscribe to the incoming MAP_REMOVE operation on a LiveMap', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + const subscriptionPromise = new Promise((resolve, reject) => + map.subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { stringKey: 'deleted' } }, + 'Check map subscription callback is called with an expected update object for MAP_REMOVE operation', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapRemoveOp({ + objectId: sampleMapObjectId, + key: 'stringKey', + }), + ); + + await subscriptionPromise; + }, + }, + + { + description: 'can subscribe to multiple incoming operations on a LiveMap', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + const expectedMapUpdates = [ + { update: { foo: 'updated' } }, + { update: { bar: 'updated' } }, + { update: { foo: 'deleted' } }, + { update: { baz: 'updated' } }, + { update: { bar: 'deleted' } }, + ]; + let currentUpdateIndex = 0; + + const subscriptionPromise = new Promise((resolve, reject) => + map.subscribe((update) => { + try { + expect(update).to.deep.equal( + expectedMapUpdates[currentUpdateIndex], + `Check map subscription callback is called with an expected update object for ${currentUpdateIndex + 1} times`, + ); + + if (currentUpdateIndex === expectedMapUpdates.length - 1) { + resolve(); + } + + currentUpdateIndex++; + } catch (error) { + reject(error); + } + }), + ); + + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapSetOp({ + objectId: sampleMapObjectId, + key: 'foo', + data: { value: 'something' }, + }), + ); + + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapSetOp({ + objectId: sampleMapObjectId, + key: 'bar', + data: { value: 'something' }, + }), + ); + + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapRemoveOp({ + objectId: sampleMapObjectId, + key: 'foo', + }), + ); + + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapSetOp({ + objectId: sampleMapObjectId, + key: 'baz', + data: { value: 'something' }, + }), + ); + + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapRemoveOp({ + objectId: sampleMapObjectId, + key: 'bar', + }), + ); + + await subscriptionPromise; + }, + }, + + { + description: 'can unsubscribe from LiveCounter updates via returned "unsubscribe" callback', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + + const counter = root.get(sampleCounterKey); + let callbackCalled = 0; + const subscriptionPromise = new Promise((resolve) => { + const { unsubscribe } = counter.subscribe(() => { + callbackCalled++; + // unsubscribe from future updates after the first call + unsubscribe(); + resolve(); + }); + }); + + const increments = 3; + for (let i = 0; i < increments; i++) { + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.counterIncOp({ + objectId: sampleCounterObjectId, + amount: 1, + }), + ); + } + + await subscriptionPromise; + + expect(counter.value()).to.equal(3, 'Check counter has final expected value after all increments'); + expect(callbackCalled).to.equal(1, 'Check subscription callback was only called once'); + }, + }, + + { + description: 'can unsubscribe from LiveCounter updates via LiveCounter.unsubscribe() call', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + + const counter = root.get(sampleCounterKey); + let callbackCalled = 0; + const subscriptionPromise = new Promise((resolve) => { + const listener = () => { + callbackCalled++; + // unsubscribe from future updates after the first call + counter.unsubscribe(listener); + resolve(); + }; + + counter.subscribe(listener); + }); + + const increments = 3; + for (let i = 0; i < increments; i++) { + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.counterIncOp({ + objectId: sampleCounterObjectId, + amount: 1, + }), + ); + } + + await subscriptionPromise; + + expect(counter.value()).to.equal(3, 'Check counter has final expected value after all increments'); + expect(callbackCalled).to.equal(1, 'Check subscription callback was only called once'); + }, + }, + + { + description: 'can remove all LiveCounter update listeners via LiveCounter.unsubscribeAll() call', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + + const counter = root.get(sampleCounterKey); + const callbacks = 3; + const callbacksCalled = new Array(callbacks).fill(0); + const subscriptionPromises = []; + + for (let i = 0; i < callbacks; i++) { + const promise = new Promise((resolve) => { + counter.subscribe(() => { + callbacksCalled[i]++; + resolve(); + }); + }); + subscriptionPromises.push(promise); + } + + const increments = 3; + for (let i = 0; i < increments; i++) { + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.counterIncOp({ + objectId: sampleCounterObjectId, + amount: 1, + }), + ); + + if (i === 0) { + // unsub all after first operation + counter.unsubscribeAll(); + } + } + + await Promise.all(subscriptionPromises); + + expect(counter.value()).to.equal(3, 'Check counter has final expected value after all increments'); + callbacksCalled.forEach((x) => expect(x).to.equal(1, 'Check subscription callbacks were called once each')); + }, + }, + + { + description: 'can unsubscribe from LiveMap updates via returned "unsubscribe" callback', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + let callbackCalled = 0; + const subscriptionPromise = new Promise((resolve) => { + const { unsubscribe } = map.subscribe(() => { + callbackCalled++; + // unsubscribe from future updates after the first call + unsubscribe(); + resolve(); + }); + }); + + const mapSets = 3; + for (let i = 0; i < mapSets; i++) { + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapSetOp({ + objectId: sampleMapObjectId, + key: `foo-${i}`, + data: { value: 'exists' }, + }), + ); + } + + await subscriptionPromise; + + for (let i = 0; i < mapSets; i++) { + expect(map.get(`foo-${i}`)).to.equal( + 'exists', + `Check map has value for key "foo-${i}" after all map sets`, + ); + } + expect(callbackCalled).to.equal(1, 'Check subscription callback was only called once'); + }, + }, + + { + description: 'can unsubscribe from LiveMap updates via LiveMap.unsubscribe() call', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + let callbackCalled = 0; + const subscriptionPromise = new Promise((resolve) => { + const listener = () => { + callbackCalled++; + // unsubscribe from future updates after the first call + map.unsubscribe(listener); + resolve(); + }; + + map.subscribe(listener); + }); + + const mapSets = 3; + for (let i = 0; i < mapSets; i++) { + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapSetOp({ + objectId: sampleMapObjectId, + key: `foo-${i}`, + data: { value: 'exists' }, + }), + ); + } + + await subscriptionPromise; + + for (let i = 0; i < mapSets; i++) { + expect(map.get(`foo-${i}`)).to.equal( + 'exists', + `Check map has value for key "foo-${i}" after all map sets`, + ); + } + expect(callbackCalled).to.equal(1, 'Check subscription callback was only called once'); + }, + }, + + { + description: 'can remove all LiveMap update listeners via LiveMap.unsubscribeAll() call', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + const callbacks = 3; + const callbacksCalled = new Array(callbacks).fill(0); + const subscriptionPromises = []; + + for (let i = 0; i < callbacks; i++) { + const promise = new Promise((resolve) => { + map.subscribe(() => { + callbacksCalled[i]++; + resolve(); + }); + }); + subscriptionPromises.push(promise); + } + + const mapSets = 3; + for (let i = 0; i < mapSets; i++) { + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapSetOp({ + objectId: sampleMapObjectId, + key: `foo-${i}`, + data: { value: 'exists' }, + }), + ); + + if (i === 0) { + // unsub all after first operation + map.unsubscribeAll(); + } + } + + await Promise.all(subscriptionPromises); + + for (let i = 0; i < mapSets; i++) { + expect(map.get(`foo-${i}`)).to.equal( + 'exists', + `Check map has value for key "foo-${i}" after all map sets`, + ); + } + callbacksCalled.forEach((x) => expect(x).to.equal(1, 'Check subscription callbacks were called once each')); + }, + }, + ]; + + for (const scenario of subscriptionCallbacksScenarios) { + if (scenario.skip === true) { + continue; + } + + /** @nospec */ + it(scenario.description, async function () { + const helper = this.test.helper; + const liveObjectsHelper = new LiveObjectsHelper(helper); + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channelName = scenario.description; + const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + const sampleMapKey = 'sampleMap'; + const sampleCounterKey = 'sampleCounter'; + + // prepare map and counter objects for use by the scenario + const { objectId: sampleMapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: sampleMapKey, + createOp: liveObjectsHelper.mapCreateOp(), + }); + const { objectId: sampleCounterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: sampleCounterKey, + createOp: liveObjectsHelper.counterCreateOp(), + }); + + await scenario.action({ + root, + liveObjectsHelper, + channelName, + channel, + sampleMapKey, + sampleMapObjectId, + sampleCounterKey, + sampleCounterObjectId, + }); + }, client); + }); + } }); /** @nospec */ From 4bca1c797caa207de96f3ff767d916f2426489b8 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 5 Nov 2024 06:51:13 +0000 Subject: [PATCH 061/166] Update moduleReport to include `external` field for LiveObjects plugin --- scripts/moduleReport.ts | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index 083f18ab9b..0eae4017a2 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -37,6 +37,18 @@ const functions = [ { name: 'constructPresenceMessage', transitiveImports: [] }, ]; +// List of all buildable plugins available as a separate export +interface PluginInfo { + description: string; + path: string; + external?: string[]; +} + +const buildablePlugins: Record<'push' | 'liveObjects', PluginInfo> = { + push: { description: 'Push', path: './build/push.js', external: ['ulid'] }, + liveObjects: { description: 'LiveObjects', path: './build/liveobjects.js', external: ['deep-equal'] }, +}; + function formatBytes(bytes: number) { const kibibytes = bytes / 1024; const formatted = kibibytes.toFixed(2); @@ -70,7 +82,7 @@ function getModularBundleInfo(exports: string[]): BundleInfo { } // Uses esbuild to create a bundle containing the named exports from a given module -function getBundleInfo(modulePath: string, exports?: string[]): BundleInfo { +function getBundleInfo(modulePath: string, exports?: string[], external?: string[]): BundleInfo { const outfile = exports ? exports.join('') : 'all'; const exportTarget = exports ? `{ ${exports.join(', ')} }` : '*'; const result = esbuild.buildSync({ @@ -84,7 +96,7 @@ function getBundleInfo(modulePath: string, exports?: string[]): BundleInfo { outfile, write: false, sourcemap: 'external', - external: ['ulid'], + external, }); const pathHasBase = (component: string) => { @@ -183,9 +195,9 @@ async function calculateAndCheckFunctionSizes(): Promise { return output; } -async function calculatePluginSize(options: { path: string; description: string }): Promise { +async function calculatePluginSize(options: PluginInfo): Promise { const output: Output = { tableRows: [], errors: [] }; - const pluginBundleInfo = getBundleInfo(options.path); + const pluginBundleInfo = getBundleInfo(options.path, undefined, options.external); const sizes = { rawByteSize: pluginBundleInfo.byteSize, gzipEncodedByteSize: (await promisify(gzip)(pluginBundleInfo.code)).byteLength, @@ -200,11 +212,11 @@ async function calculatePluginSize(options: { path: string; description: string } async function calculatePushPluginSize(): Promise { - return calculatePluginSize({ path: './build/push.js', description: 'Push' }); + return calculatePluginSize(buildablePlugins.push); } async function calculateLiveObjectsPluginSize(): Promise { - return calculatePluginSize({ path: './build/liveobjects.js', description: 'LiveObjects' }); + return calculatePluginSize(buildablePlugins.liveObjects); } async function calculateAndCheckMinimalUsefulRealtimeBundleSize(): Promise { @@ -291,7 +303,8 @@ async function checkBaseRealtimeFiles() { } async function checkPushPluginFiles() { - const pushPluginBundleInfo = getBundleInfo('./build/push.js'); + const { path, external } = buildablePlugins.push; + const pushPluginBundleInfo = getBundleInfo(path, undefined, external); // These are the files that are allowed to contribute >= `threshold` bytes to the Push bundle. const allowedFiles = new Set([ @@ -305,7 +318,8 @@ async function checkPushPluginFiles() { } async function checkLiveObjectsPluginFiles() { - const pluginBundleInfo = getBundleInfo('./build/liveobjects.js'); + const { path, external } = buildablePlugins.liveObjects; + const pluginBundleInfo = getBundleInfo(path, undefined, external); // These are the files that are allowed to contribute >= `threshold` bytes to the LiveObjects bundle. const allowedFiles = new Set([ From 924eafd85546f0d54f768d7d9257c54fcd2bd161 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 22 Nov 2024 16:57:20 +0000 Subject: [PATCH 062/166] Fix LiveObjects subscription tests were expecting incorrect `deleted` update object for MAP_REMOVE ops --- test/realtime/live_objects.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 9353245270..5a9eda3aa1 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -1351,7 +1351,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], map.subscribe((update) => { try { expect(update).to.deep.equal( - { update: { stringKey: 'deleted' } }, + { update: { stringKey: 'removed' } }, 'Check map subscription callback is called with an expected update object for MAP_REMOVE operation', ); resolve(); @@ -1382,9 +1382,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const expectedMapUpdates = [ { update: { foo: 'updated' } }, { update: { bar: 'updated' } }, - { update: { foo: 'deleted' } }, + { update: { foo: 'removed' } }, { update: { baz: 'updated' } }, - { update: { bar: 'deleted' } }, + { update: { bar: 'removed' } }, ]; let currentUpdateIndex = 0; From f59c4595e06407978600575eb1fcda5aba65a782 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 22 Nov 2024 16:45:09 +0000 Subject: [PATCH 063/166] Refactor scenario tests in LiveObjects tests --- test/realtime/live_objects.test.js | 45 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 5a9eda3aa1..9e75d9c542 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -30,6 +30,21 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(object.constructor.name).to.match(new RegExp(`_?${className}`), msg); } + function forScenarios(scenarios, testFn) { + // if there are scenarios marked as "only", run only them. + // otherwise go over every scenario + const onlyScenarios = scenarios.filter((x) => x.only === true); + const scenariosToRun = onlyScenarios.length > 0 ? onlyScenarios : scenarios; + + for (const scenario of scenariosToRun) { + if (scenario.skip === true) { + continue; + } + + testFn(scenario); + } + } + describe('realtime/live_objects', function () { this.timeout(60 * 1000); @@ -881,11 +896,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, ]; - for (const scenario of applyOperationsScenarios) { - if (scenario.skip === true) { - continue; - } - + forScenarios(applyOperationsScenarios, (scenario) => /** @nospec */ it(`can apply ${scenario.description} state operation messages`, async function () { const helper = this.test.helper; @@ -902,8 +913,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await scenario.action({ root, liveObjectsHelper, channelName }); }, client); - }); - } + }), + ); const applyOperationsDuringSyncScenarios = [ { @@ -1206,11 +1217,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, ]; - for (const scenario of applyOperationsDuringSyncScenarios) { - if (scenario.skip === true) { - continue; - } - + forScenarios(applyOperationsDuringSyncScenarios, (scenario) => /** @nospec */ it(scenario.description, async function () { const helper = this.test.helper; @@ -1229,8 +1236,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await scenario.action({ root, liveObjectsHelper, channelName, channel }); }, client); - }); - } + }), + ); const subscriptionCallbacksScenarios = [ { @@ -1699,11 +1706,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, ]; - for (const scenario of subscriptionCallbacksScenarios) { - if (scenario.skip === true) { - continue; - } - + forScenarios(subscriptionCallbacksScenarios, (scenario) => /** @nospec */ it(scenario.description, async function () { const helper = this.test.helper; @@ -1744,8 +1747,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], sampleCounterObjectId, }); }, client); - }); - } + }), + ); }); /** @nospec */ From 63126e2921315839fae08304480c46595183189e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 21 Nov 2024 00:25:54 +0000 Subject: [PATCH 064/166] Add `siteCode` to Timeserial --- src/plugins/liveobjects/timeserial.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/plugins/liveobjects/timeserial.ts b/src/plugins/liveobjects/timeserial.ts index 0accdc81be..bc5c535505 100644 --- a/src/plugins/liveobjects/timeserial.ts +++ b/src/plugins/liveobjects/timeserial.ts @@ -9,6 +9,11 @@ export interface Timeserial { */ readonly seriesId: string; + /** + * The site code of the timeserial. + */ + readonly siteCode: string; + /** * The timestamp of the timeserial. */ @@ -40,6 +45,7 @@ export interface Timeserial { */ export class DefaultTimeserial implements Timeserial { public readonly seriesId: string; + public readonly siteCode: string; public readonly timestamp: number; public readonly counter: number; public readonly index?: number; @@ -55,6 +61,8 @@ export class DefaultTimeserial implements Timeserial { this.timestamp = timestamp; this.counter = counter; this.index = index; + // TODO: will be removed once https://ably.atlassian.net/browse/DTP-1078 is implemented on the realtime + this.siteCode = this.seriesId.slice(0, 3); // site code is stored in the first 3 letters of the epoch, which is stored in the series id field } /** From 8ba4b940f5b44668a3ef16534268b85f02462485 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 21 Nov 2024 04:07:46 +0000 Subject: [PATCH 065/166] Handle site timeserials vector on StateObject Resolves DTP-1077 --- src/common/lib/client/realtimechannel.ts | 2 +- src/plugins/liveobjects/livecounter.ts | 27 +- src/plugins/liveobjects/livemap.ts | 29 +- src/plugins/liveobjects/liveobject.ts | 34 +- src/plugins/liveobjects/liveobjects.ts | 37 +- src/plugins/liveobjects/liveobjectspool.ts | 82 +-- src/plugins/liveobjects/statemessage.ts | 4 +- .../liveobjects/syncliveobjectsdatapool.ts | 20 +- test/common/modules/live_objects_helper.js | 8 +- test/realtime/live_objects.test.js | 513 +++++++++++++++--- 10 files changed, 546 insertions(+), 210 deletions(-) diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index b7ea4e577a..4cdcd23126 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -649,7 +649,7 @@ class RealtimeChannel extends EventEmitter { } } - this._liveObjects.handleStateMessages(stateMessages, message.channelSerial); + this._liveObjects.handleStateMessages(stateMessages); break; } diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 5ec413434b..0327adafcb 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -1,7 +1,7 @@ import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; import { StateCounter, StateCounterOp, StateMessage, StateOperation, StateOperationAction } from './statemessage'; -import { Timeserial } from './timeserial'; +import { DefaultTimeserial, Timeserial } from './timeserial'; export interface LiveCounterData extends LiveObjectData { data: number; @@ -17,9 +17,9 @@ export class LiveCounter extends LiveObject private _created: boolean, initialData?: LiveCounterData | null, objectId?: string, - regionalTimeserial?: Timeserial, + siteTimeserials?: Record, ) { - super(liveObjects, initialData, objectId, regionalTimeserial); + super(liveObjects, initialData, objectId, siteTimeserials); } /** @@ -31,9 +31,9 @@ export class LiveCounter extends LiveObject liveobjects: LiveObjects, isCreated: boolean, objectId?: string, - regionalTimeserial?: Timeserial, + siteTimeserials?: Record, ): LiveCounter { - return new LiveCounter(liveobjects, isCreated, null, objectId, regionalTimeserial); + return new LiveCounter(liveobjects, isCreated, null, objectId, siteTimeserials); } value(): number { @@ -57,7 +57,7 @@ export class LiveCounter extends LiveObject /** * @internal */ - applyOperation(op: StateOperation, msg: StateMessage, opRegionalTimeserial: Timeserial): void { + applyOperation(op: StateOperation, msg: StateMessage): void { if (op.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( `Cannot apply state operation with objectId=${op.objectId}, to this LiveCounter with objectId=${this.getObjectId()}`, @@ -66,6 +66,20 @@ export class LiveCounter extends LiveObject ); } + const opOriginTimeserial = DefaultTimeserial.calculateTimeserial(this._client, msg.serial); + if (!this._canApplyOperation(opOriginTimeserial)) { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveCounter.applyOperation()', + `skipping ${op.action} op: op timeserial ${opOriginTimeserial.toString()} <= site timeserial ${this._siteTimeserials[opOriginTimeserial.siteCode].toString()}; objectId=${this._objectId}`, + ); + return; + } + // should update stored site timeserial immediately. doesn't matter if we successfully apply the op, + // as it's important to mark that the op was processed by the object + this._siteTimeserials[opOriginTimeserial.siteCode] = opOriginTimeserial; + let update: LiveCounterUpdate | LiveObjectUpdateNoop; switch (op.action) { case StateOperationAction.COUNTER_CREATE: @@ -90,7 +104,6 @@ export class LiveCounter extends LiveObject ); } - this.setRegionalTimeserial(opRegionalTimeserial); this.notifyUpdated(update); } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 97129f8964..de19baa961 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -52,9 +52,9 @@ export class LiveMap extends LiveObject { private _semantics: MapSemantics, initialData?: LiveMapData | null, objectId?: string, - regionalTimeserial?: Timeserial, + siteTimeserials?: Record, ) { - super(liveObjects, initialData, objectId, regionalTimeserial); + super(liveObjects, initialData, objectId, siteTimeserials); } /** @@ -62,8 +62,8 @@ export class LiveMap extends LiveObject { * * @internal */ - static zeroValue(liveobjects: LiveObjects, objectId?: string, regionalTimeserial?: Timeserial): LiveMap { - return new LiveMap(liveobjects, MapSemantics.LWW, null, objectId, regionalTimeserial); + static zeroValue(liveobjects: LiveObjects, objectId?: string, siteTimeserials?: Record): LiveMap { + return new LiveMap(liveobjects, MapSemantics.LWW, null, objectId, siteTimeserials); } /** @@ -144,7 +144,7 @@ export class LiveMap extends LiveObject { /** * @internal */ - applyOperation(op: StateOperation, msg: StateMessage, opRegionalTimeserial: Timeserial): void { + applyOperation(op: StateOperation, msg: StateMessage): void { if (op.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( `Cannot apply state operation with objectId=${op.objectId}, to this LiveMap with objectId=${this.getObjectId()}`, @@ -153,6 +153,20 @@ export class LiveMap extends LiveObject { ); } + const opOriginTimeserial = DefaultTimeserial.calculateTimeserial(this._client, msg.serial); + if (!this._canApplyOperation(opOriginTimeserial)) { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveMap.applyOperation()', + `skipping ${op.action} op: op timeserial ${opOriginTimeserial.toString()} <= site timeserial ${this._siteTimeserials[opOriginTimeserial.siteCode].toString()}; objectId=${this._objectId}`, + ); + return; + } + // should update stored site timeserial immediately. doesn't matter if we successfully apply the op, + // as it's important to mark that the op was processed by the object + this._siteTimeserials[opOriginTimeserial.siteCode] = opOriginTimeserial; + let update: LiveMapUpdate | LiveObjectUpdateNoop; switch (op.action) { case StateOperationAction.MAP_CREATE: @@ -165,7 +179,7 @@ export class LiveMap extends LiveObject { // leave an explicit return here, so that TS knows that update object is always set after the switch statement. return; } else { - update = this._applyMapSet(op.mapOp, DefaultTimeserial.calculateTimeserial(this._client, msg.serial)); + update = this._applyMapSet(op.mapOp, opOriginTimeserial); } break; @@ -175,7 +189,7 @@ export class LiveMap extends LiveObject { // leave an explicit return here, so that TS knows that update object is always set after the switch statement. return; } else { - update = this._applyMapRemove(op.mapOp, DefaultTimeserial.calculateTimeserial(this._client, msg.serial)); + update = this._applyMapRemove(op.mapOp, opOriginTimeserial); } break; @@ -187,7 +201,6 @@ export class LiveMap extends LiveObject { ); } - this.setRegionalTimeserial(opRegionalTimeserial); this.notifyUpdated(update); } diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index 78244ab650..e18762dbd8 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -2,7 +2,7 @@ import type BaseClient from 'common/lib/client/baseclient'; import type EventEmitter from 'common/lib/util/eventemitter'; import { LiveObjects } from './liveobjects'; import { StateMessage, StateOperation } from './statemessage'; -import { DefaultTimeserial, Timeserial } from './timeserial'; +import { Timeserial } from './timeserial'; enum LiveObjectEvents { Updated = 'Updated', @@ -34,20 +34,20 @@ export abstract class LiveObject< protected _eventEmitter: EventEmitter; protected _dataRef: TData; protected _objectId: string; - protected _regionalTimeserial: Timeserial; + protected _siteTimeserials: Record; constructor( protected _liveObjects: LiveObjects, initialData?: TData | null, objectId?: string, - regionalTimeserial?: Timeserial, + siteTimeserials?: Record, ) { this._client = this._liveObjects.getClient(); this._eventEmitter = new this._client.EventEmitter(this._client.logger); this._dataRef = initialData ?? this._getZeroValueData(); this._objectId = objectId ?? this._createObjectId(); - // use zero value timeserial by default, so any future operation can be applied for this object - this._regionalTimeserial = regionalTimeserial ?? DefaultTimeserial.zeroValueTimeserial(this._client); + // use empty timeserials vector by default, so any future operation can be applied to this object + this._siteTimeserials = siteTimeserials ?? {}; } subscribe(listener: (update: TUpdate) => void): SubscribeResponse { @@ -82,13 +82,6 @@ export abstract class LiveObject< return this._objectId; } - /** - * @internal - */ - getRegionalTimeserial(): Timeserial { - return this._regionalTimeserial; - } - /** * Sets a new data reference for the LiveObject and returns an update object that describes the changes applied based on the object's previous value. * @@ -103,8 +96,8 @@ export abstract class LiveObject< /** * @internal */ - setRegionalTimeserial(regionalTimeserial: Timeserial): void { - this._regionalTimeserial = regionalTimeserial; + setSiteTimeserials(siteTimeserials: Record): void { + this._siteTimeserials = siteTimeserials; } /** @@ -119,6 +112,17 @@ export abstract class LiveObject< this._eventEmitter.emit(LiveObjectEvents.Updated, update); } + /** + * Returns true if the given origin timeserial indicates that the operation to which it belongs should be applied to the object. + * + * An operation should be applied if the origin timeserial is strictly greater than the timeserial in the site timeserials for the same site. + * If the site timeserials do not contain a timeserial for the site of the origin timeserial, the operation should be applied. + */ + protected _canApplyOperation(opOriginTimeserial: Timeserial): boolean { + const siteTimeserial = this._siteTimeserials[opOriginTimeserial.siteCode]; + return !siteTimeserial || opOriginTimeserial.after(siteTimeserial); + } + private _createObjectId(): string { // TODO: implement object id generation based on live object type and initial value return Math.random().toString().substring(2); @@ -127,7 +131,7 @@ export abstract class LiveObject< /** * @internal */ - abstract applyOperation(op: StateOperation, msg: StateMessage, opRegionalTimeserial: Timeserial): void; + abstract applyOperation(op: StateOperation, msg: StateMessage): void; protected abstract _getZeroValueData(): TData; /** * Calculate the update object based on the current Live Object data and incoming new data. diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index a5070ec993..9791498e0c 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -8,17 +8,11 @@ import { LiveObject, LiveObjectUpdate } from './liveobject'; import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage } from './statemessage'; import { LiveCounterDataEntry, SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; -import { DefaultTimeserial, Timeserial } from './timeserial'; enum LiveObjectsEvents { SyncCompleted = 'SyncCompleted', } -export interface BufferedStateMessage { - stateMessage: StateMessage; - regionalTimeserial: Timeserial; -} - export class LiveObjects { private _client: BaseClient; private _channel: RealtimeChannel; @@ -30,7 +24,7 @@ export class LiveObjects { private _syncInProgress: boolean; private _currentSyncId: string | undefined; private _currentSyncCursor: string | undefined; - private _bufferedStateOperations: BufferedStateMessage[]; + private _bufferedStateOperations: StateMessage[]; constructor(channel: RealtimeChannel) { this._channel = channel; @@ -92,23 +86,17 @@ export class LiveObjects { /** * @internal */ - handleStateMessages(stateMessages: StateMessage[], msgRegionalTimeserial: string | null | undefined): void { - const timeserial = DefaultTimeserial.calculateTimeserial(this._client, msgRegionalTimeserial); - + handleStateMessages(stateMessages: StateMessage[]): void { if (this._syncInProgress) { // The client receives state messages in realtime over the channel concurrently with the SYNC sequence. // Some of the incoming state messages may have already been applied to the state objects described in // the SYNC sequence, but others may not; therefore we must buffer these messages so that we can apply - // them to the state objects once the SYNC is complete. To avoid double-counting, the buffered operations - // are applied according to the state object's regional timeserial, which reflects the regional timeserial - // of the state message that was last applied to that state object. - stateMessages.forEach((x) => - this._bufferedStateOperations.push({ stateMessage: x, regionalTimeserial: timeserial }), - ); + // them to the state objects once the SYNC is complete. + this._bufferedStateOperations.push(...stateMessages); return; } - this._liveObjectsPool.applyStateMessages(stateMessages, timeserial); + this._liveObjectsPool.applyStateMessages(stateMessages); } /** @@ -164,8 +152,9 @@ export class LiveObjects { private _endSync(): void { this._applySync(); - // should apply buffered state operations after we applied the SYNC data - this._liveObjectsPool.applyBufferedStateMessages(this._bufferedStateOperations); + // should apply buffered state operations after we applied the SYNC data. + // can use regular state messages application logic + this._liveObjectsPool.applyStateMessages(this._bufferedStateOperations); this._bufferedStateOperations = []; this._syncLiveObjectsDataPool.reset(); @@ -204,11 +193,13 @@ export class LiveObjects { for (const [objectId, entry] of this._syncLiveObjectsDataPool.entries()) { receivedObjectIds.add(objectId); const existingObject = this._liveObjectsPool.get(objectId); - const regionalTimeserialObj = DefaultTimeserial.calculateTimeserial(this._client, entry.regionalTimeserial); if (existingObject) { + // SYNC sequence is a source of truth for the current state of the objects, + // so we can use the data received from the SYNC sequence directly + // without the need to merge data values or site timeserials. const update = existingObject.setData(entry.objectData); - existingObject.setRegionalTimeserial(regionalTimeserialObj); + existingObject.setSiteTimeserials(entry.siteTimeserials); if (existingObject instanceof LiveCounter) { existingObject.setCreated((entry as LiveCounterDataEntry).created); } @@ -223,11 +214,11 @@ export class LiveObjects { const objectType = entry.objectType; switch (objectType) { case 'LiveCounter': - newObject = new LiveCounter(this, entry.created, entry.objectData, objectId, regionalTimeserialObj); + newObject = new LiveCounter(this, entry.created, entry.objectData, objectId, entry.siteTimeserials); break; case 'LiveMap': - newObject = new LiveMap(this, entry.semantics, entry.objectData, objectId, regionalTimeserialObj); + newObject = new LiveMap(this, entry.semantics, entry.objectData, objectId, entry.siteTimeserials); break; default: diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index a4ea8c92ef..22f1edb943 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -3,7 +3,7 @@ import type RealtimeChannel from 'common/lib/client/realtimechannel'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; -import { BufferedStateMessage, LiveObjects } from './liveobjects'; +import { LiveObjects } from './liveobjects'; import { ObjectId } from './objectid'; import { MapSemantics, StateMessage, StateOperation, StateOperationAction } from './statemessage'; import { DefaultTimeserial, Timeserial } from './timeserial'; @@ -52,24 +52,22 @@ export class LiveObjectsPool { } const parsedObjectId = ObjectId.fromString(this._client, objectId); - // use zero value timeserial, so any operation can be applied for this object - const regionalTimeserial = DefaultTimeserial.zeroValueTimeserial(this._client); let zeroValueObject: LiveObject; switch (parsedObjectId.type) { case 'map': { - zeroValueObject = LiveMap.zeroValue(this._liveObjects, objectId, regionalTimeserial); + zeroValueObject = LiveMap.zeroValue(this._liveObjects, objectId); break; } case 'counter': - zeroValueObject = LiveCounter.zeroValue(this._liveObjects, false, objectId, regionalTimeserial); + zeroValueObject = LiveCounter.zeroValue(this._liveObjects, false, objectId); break; } this.set(objectId, zeroValueObject); } - applyStateMessages(stateMessages: StateMessage[], regionalTimeserial: Timeserial): void { + applyStateMessages(stateMessages: StateMessage[]): void { for (const stateMessage of stateMessages) { if (!stateMessage.operation) { this._client.Logger.logAction( @@ -81,6 +79,7 @@ export class LiveObjectsPool { continue; } + const opOriginTimeserial = DefaultTimeserial.calculateTimeserial(this._client, stateMessage.serial); const stateOperation = stateMessage.operation; switch (stateOperation.action) { @@ -89,17 +88,17 @@ export class LiveObjectsPool { if (this.get(stateOperation.objectId)) { // object wich such id already exists (we may have created a zero-value object before, or this is a duplicate *_CREATE op), // so delegate application of the op to that object - this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage, regionalTimeserial); + this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); break; } // otherwise we can create new objects in the pool if (stateOperation.action === StateOperationAction.MAP_CREATE) { - this._handleMapCreate(stateOperation, regionalTimeserial); + this._handleMapCreate(stateOperation, opOriginTimeserial); } if (stateOperation.action === StateOperationAction.COUNTER_CREATE) { - this._handleCounterCreate(stateOperation, regionalTimeserial); + this._handleCounterCreate(stateOperation, opOriginTimeserial); } break; @@ -110,7 +109,7 @@ export class LiveObjectsPool { // we create a zero-value object for the provided object id, and apply operation for that zero-value object. // when we eventually receive a corresponding *_CREATE op for that object, its application will be handled by that zero-value object. this.createZeroValueObjectIfNotExists(stateOperation.objectId); - this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage, regionalTimeserial); + this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); break; default: @@ -124,49 +123,6 @@ export class LiveObjectsPool { } } - applyBufferedStateMessages(bufferedStateMessages: BufferedStateMessage[]): void { - // since we receive state operation messages concurrently with the SYNC sequence, - // we must determine which operation messages should be applied to the now local copy of the object pool, and the rest will be skipped. - // since messages are delivered in regional order to the client, we can inspect the regional timeserial - // of each state operation message to know whether it has reached a point in the message stream - // that is no longer included in the state object snapshot we received from SYNC sequence. - for (const { regionalTimeserial, stateMessage } of bufferedStateMessages) { - if (!stateMessage.operation) { - this._client.Logger.logAction( - this._client.logger, - this._client.Logger.LOG_MAJOR, - 'LiveObjects.LiveObjectsPool.applyBufferedStateMessages()', - `state operation message is received without 'operation' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, - ); - continue; - } - - const existingObject = this.get(stateMessage.operation.objectId); - if (!existingObject) { - // for object ids we haven't seen yet we can apply operation immediately - this.applyStateMessages([stateMessage], regionalTimeserial); - continue; - } - - // otherwise we need to compare regional timeserials - if ( - regionalTimeserial.before(existingObject.getRegionalTimeserial()) || - regionalTimeserial.equal(existingObject.getRegionalTimeserial()) - ) { - // the operation's regional timeserial <= the object's timeserial, ignore the operation. - this._client.Logger.logAction( - this._client.logger, - this._client.Logger.LOG_MICRO, - 'LiveObjects.LiveObjectsPool.applyBufferedStateMessages()', - `skipping buffered state operation message: op regional timeserial ${regionalTimeserial.toString()} <= object regional timeserial ${existingObject.getRegionalTimeserial().toString()}; objectId=${stateMessage.operation.objectId}, message id: ${stateMessage.id}, channel: ${this._channel.name}`, - ); - continue; - } - - this.applyStateMessages([stateMessage], regionalTimeserial); - } - } - private _getInitialPool(): Map { const pool = new Map(); const root = LiveMap.zeroValue(this._liveObjects, ROOT_OBJECT_ID); @@ -174,29 +130,37 @@ export class LiveObjectsPool { return pool; } - private _handleCounterCreate(stateOperation: StateOperation, opRegionalTimeserial: Timeserial): void { + private _handleCounterCreate(stateOperation: StateOperation, opOriginTimeserial: Timeserial): void { + // should use op's origin timeserial as the initial value for the object's site timeserials vector + const siteTimeserials = { + [opOriginTimeserial.siteCode]: opOriginTimeserial, + }; let counter: LiveCounter; if (this._client.Utils.isNil(stateOperation.counter)) { // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly a zero-value counter. - counter = LiveCounter.zeroValue(this._liveObjects, true, stateOperation.objectId, opRegionalTimeserial); + counter = LiveCounter.zeroValue(this._liveObjects, true, stateOperation.objectId, siteTimeserials); } else { counter = new LiveCounter( this._liveObjects, true, { data: stateOperation.counter.count ?? 0 }, stateOperation.objectId, - opRegionalTimeserial, + siteTimeserials, ); } this.set(stateOperation.objectId, counter); } - private _handleMapCreate(stateOperation: StateOperation, opRegionalTimeserial: Timeserial): void { + private _handleMapCreate(stateOperation: StateOperation, opOriginTimeserial: Timeserial): void { + // should use op's origin timeserial as the initial value for the object's site timeserials vector + const siteTimeserials = { + [opOriginTimeserial.siteCode]: opOriginTimeserial, + }; let map: LiveMap; if (this._client.Utils.isNil(stateOperation.map)) { // if a map object is missing for the MAP_CREATE op, the initial value is implicitly a zero-value map. - map = LiveMap.zeroValue(this._liveObjects, stateOperation.objectId, opRegionalTimeserial); + map = LiveMap.zeroValue(this._liveObjects, stateOperation.objectId, siteTimeserials); } else { const objectData = LiveMap.liveMapDataFromMapEntries(this._client, stateOperation.map.entries ?? {}); map = new LiveMap( @@ -204,7 +168,7 @@ export class LiveObjectsPool { stateOperation.map.semantics ?? MapSemantics.LWW, objectData, stateOperation.objectId, - opRegionalTimeserial, + siteTimeserials, ); } diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index b4eefdadd0..d3af8bd3d6 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -110,8 +110,8 @@ export interface StateOperation { export interface StateObject { /** The identifier of the state object. */ objectId: string; - /** The *regional* timeserial of the last operation that was applied to this state object. */ - regionalTimeserial: string; + /** A vector of origin timeserials keyed by site code of the last operation that was applied to this state object. */ + siteTimeserials: Record; /** The data that represents the state of the object if it is a Map object type. */ map?: StateMap; /** The data that represents the state of the object if it is a Counter object type. */ diff --git a/src/plugins/liveobjects/syncliveobjectsdatapool.ts b/src/plugins/liveobjects/syncliveobjectsdatapool.ts index 8f0f8d6485..194c3165c0 100644 --- a/src/plugins/liveobjects/syncliveobjectsdatapool.ts +++ b/src/plugins/liveobjects/syncliveobjectsdatapool.ts @@ -5,10 +5,11 @@ import { LiveMap } from './livemap'; import { LiveObjectData } from './liveobject'; import { LiveObjects } from './liveobjects'; import { MapSemantics, StateMessage, StateObject } from './statemessage'; +import { DefaultTimeserial, Timeserial } from './timeserial'; export interface LiveObjectDataEntry { objectData: LiveObjectData; - regionalTimeserial: string; + siteTimeserials: Record; objectType: 'LiveMap' | 'LiveCounter'; } @@ -92,7 +93,7 @@ export class SyncLiveObjectsDataPool { const newEntry: LiveCounterDataEntry = { objectData, objectType: 'LiveCounter', - regionalTimeserial: stateObject.regionalTimeserial, + siteTimeserials: this._timeserialMapFromStringMap(stateObject.siteTimeserials), created: counter.created, }; @@ -106,10 +107,23 @@ export class SyncLiveObjectsDataPool { const newEntry: LiveMapDataEntry = { objectData, objectType: 'LiveMap', - regionalTimeserial: stateObject.regionalTimeserial, + siteTimeserials: this._timeserialMapFromStringMap(stateObject.siteTimeserials), semantics: map.semantics ?? MapSemantics.LWW, }; return newEntry; } + + private _timeserialMapFromStringMap(stringTimeserialsMap: Record): Record { + const objTimeserialsMap = Object.entries(stringTimeserialsMap).reduce( + (acc, v) => { + const [key, timeserialString] = v; + acc[key] = DefaultTimeserial.calculateTimeserial(this._client, timeserialString); + return acc; + }, + {} as Record, + ); + + return objTimeserialsMap; + } } diff --git a/test/common/modules/live_objects_helper.js b/test/common/modules/live_objects_helper.js index 82c65b7377..9f8e98783c 100644 --- a/test/common/modules/live_objects_helper.js +++ b/test/common/modules/live_objects_helper.js @@ -175,11 +175,11 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb } mapObject(opts) { - const { objectId, regionalTimeserial, entries } = opts; + const { objectId, siteTimeserials, entries } = opts; const obj = { object: { objectId, - regionalTimeserial, + siteTimeserials, map: { entries }, }, }; @@ -188,11 +188,11 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb } counterObject(opts) { - const { objectId, regionalTimeserial, count } = opts; + const { objectId, siteTimeserials, count } = opts; const obj = { object: { objectId, - regionalTimeserial, + siteTimeserials, counter: { created: true, count, diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 9e75d9c542..34ded67714 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -125,7 +125,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await liveObjectsHelper.processStateObjectMessageOnChannel({ channel: testChannel, syncSerial: 'serial:', - state: [liveObjectsHelper.mapObject({ objectId: 'root', regionalTimeserial: '@0-0' })], + state: [liveObjectsHelper.mapObject({ objectId: 'root', siteTimeserials: { '000': '000@0-0' } })], }); const publishChannel = publishClient.channels.get('channel'); @@ -292,8 +292,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], state: [ liveObjectsHelper.mapObject({ objectId: 'root', - regionalTimeserial: '@0-0', - entries: { key: { timeserial: '@0-0', data: { value: 1 } } }, + siteTimeserials: { '000': '000@0-0' }, + entries: { key: { timeserial: '000@0-0', data: { value: 1 } } }, }), ], }); @@ -501,7 +501,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ]; const applyOperationsScenarios = [ { - description: 'MAP_CREATE with primitives', + description: 'can apply MAP_CREATE with primitives state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -560,7 +560,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { - description: 'MAP_CREATE with object ids', + description: 'can apply MAP_CREATE with object ids state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; const withReferencesMapKey = 'withReferencesMap'; @@ -635,7 +635,86 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { - description: 'MAP_SET with primitives', + description: + 'MAP_CREATE state operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + // need to use multiple maps as MAP_CREATE op can only be applied once to a map object + const mapIds = [ + liveObjectsHelper.fakeMapObjectId(), + liveObjectsHelper.fakeMapObjectId(), + liveObjectsHelper.fakeMapObjectId(), + liveObjectsHelper.fakeMapObjectId(), + liveObjectsHelper.fakeMapObjectId(), + ]; + await Promise.all( + mapIds.map(async (mapId, i) => { + // send a MAP_SET op first to create a zero-value map with forged site timeserials vector (from the op), and set it on a root. + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: 'bbb@1-0', + state: [liveObjectsHelper.mapSetOp({ objectId: mapId, key: 'foo', data: { value: 'bar' } })], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: `aaa@${i}-0`, + state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: mapId, data: { objectId: mapId } })], + }); + }), + ); + + // inject operations with various timeserial values + for (const [i, serial] of [ + 'bbb@0-0', // existing site, earlier CGO, not applied + 'bbb@1-0', // existing site, same CGO, not applied + 'bbb@2-0', // existing site, later CGO, applied + 'aaa@0-0', // different site, earlier CGO, applied + 'ccc@9-0', // different site, later CGO, applied + ].entries()) { + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial, + state: [ + liveObjectsHelper.mapCreateOp({ + objectId: mapIds[i], + entries: { + baz: { timeserial: serial, data: { value: 'qux' } }, + }, + }), + ], + }); + } + + // check only operations with correct timeserials were applied + const expectedMapValues = [ + { foo: 'bar' }, + { foo: 'bar' }, + { foo: 'bar', baz: 'qux' }, // applied MAP_CREATE + { foo: 'bar', baz: 'qux' }, // applied MAP_CREATE + { foo: 'bar', baz: 'qux' }, // applied MAP_CREATE + ]; + + for (const [i, mapId] of mapIds.entries()) { + const expectedMapValue = expectedMapValues[i]; + const expectedKeysCount = Object.keys(expectedMapValue).length; + + expect(root.get(mapId).size()).to.equal( + expectedKeysCount, + `Check map #${i + 1} has expected number of keys after MAP_CREATE ops`, + ); + Object.entries(expectedMapValue).forEach(([key, value]) => { + expect(root.get(mapId).get(key)).to.equal( + value, + `Check map #${i + 1} has expected value for "${key}" key after MAP_CREATE ops`, + ); + }); + } + }, + }, + + { + description: 'can apply MAP_SET with primitives state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -678,7 +757,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { - description: 'MAP_SET with object ids', + description: 'can apply MAP_SET with object ids state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -729,7 +808,73 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { - description: 'MAP_REMOVE', + description: + 'MAP_SET state operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + // create new map and set it on a root with forged timeserials + const mapId = liveObjectsHelper.fakeMapObjectId(); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: 'bbb@1-0', + state: [ + liveObjectsHelper.mapCreateOp({ + objectId: mapId, + entries: { + foo1: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo2: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo3: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo4: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo5: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo6: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + }, + }), + ], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: 'aaa@0-0', + state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'map', data: { objectId: mapId } })], + }); + + // inject operations with various timeserial values + for (const [i, serial] of [ + 'bbb@0-0', // existing site, earlier site CGO, not applied + 'bbb@1-0', // existing site, same site CGO, not applied + 'bbb@2-0', // existing site, later site CGO, applied, site timeserials updated + 'bbb@2-0', // existing site, same site CGO (updated from last op), not applied + 'aaa@0-0', // different site, earlier entry CGO, not applied + 'ccc@9-0', // different site, later entry CGO, applied + ].entries()) { + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial, + state: [liveObjectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { value: 'baz' } })], + }); + } + + // check only operations with correct timeserials were applied + const expectedMapKeys = [ + { key: 'foo1', value: 'bar' }, + { key: 'foo2', value: 'bar' }, + { key: 'foo3', value: 'baz' }, // updated + { key: 'foo4', value: 'bar' }, + { key: 'foo5', value: 'bar' }, + { key: 'foo6', value: 'baz' }, // updated + ]; + + expectedMapKeys.forEach(({ key, value }) => { + expect(root.get('map').get(key)).to.equal( + value, + `Check "${key}" key on map has expected value after MAP_SET ops`, + ); + }); + }, + }, + + { + description: 'can apply MAP_REMOVE state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; const mapKey = 'map'; @@ -787,7 +932,76 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { - description: 'COUNTER_CREATE', + description: + 'MAP_REMOVE state operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + // create new map and set it on a root with forged timeserials + const mapId = liveObjectsHelper.fakeMapObjectId(); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: 'bbb@1-0', + state: [ + liveObjectsHelper.mapCreateOp({ + objectId: mapId, + entries: { + foo1: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo2: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo3: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo4: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo5: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo6: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + }, + }), + ], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: 'aaa@0-0', + state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'map', data: { objectId: mapId } })], + }); + + // inject operations with various timeserial values + for (const [i, serial] of [ + 'bbb@0-0', // existing site, earlier site CGO, not applied + 'bbb@1-0', // existing site, same site CGO, not applied + 'bbb@2-0', // existing site, later site CGO, applied, site timeserials updated + 'bbb@2-0', // existing site, same site CGO (updated from last op), not applied + 'aaa@0-0', // different site, earlier entry CGO, not applied + 'ccc@9-0', // different site, later entry CGO, applied + ].entries()) { + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial, + state: [liveObjectsHelper.mapRemoveOp({ objectId: mapId, key: `foo${i + 1}` })], + }); + } + + // check only operations with correct timeserials were applied + const expectedMapKeys = [ + { key: 'foo1', exists: true }, + { key: 'foo2', exists: true }, + { key: 'foo3', exists: false }, // removed + { key: 'foo4', exists: true }, + { key: 'foo5', exists: true }, + { key: 'foo6', exists: false }, // removed + ]; + + expectedMapKeys.forEach(({ key, exists }) => { + if (exists) { + expect(root.get('map').get(key), `Check "${key}" key on map still exists after MAP_REMOVE ops`).to + .exist; + } else { + expect(root.get('map').get(key), `Check "${key}" key on map does not exist after MAP_REMOVE ops`).to.not + .exist; + } + }); + }, + }, + + { + description: 'can apply COUNTER_CREATE state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -837,7 +1051,74 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { - description: 'COUNTER_INC', + description: + 'COUNTER_CREATE state operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + // need to use multiple counters as COUNTER_CREATE op can only be applied once to a counter object + const counterIds = [ + liveObjectsHelper.fakeCounterObjectId(), + liveObjectsHelper.fakeCounterObjectId(), + liveObjectsHelper.fakeCounterObjectId(), + liveObjectsHelper.fakeCounterObjectId(), + liveObjectsHelper.fakeCounterObjectId(), + ]; + await Promise.all( + counterIds.map(async (counterId, i) => { + // send a COUNTER_INC op first to create a zero-value counter with forged site timeserials vector (from the op), and set it on a root. + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: 'bbb@1-0', + state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: `aaa@${i}-0`, + state: [ + liveObjectsHelper.mapSetOp({ objectId: 'root', key: counterId, data: { objectId: counterId } }), + ], + }); + }), + ); + + // inject operations with various timeserial values + for (const [i, serial] of [ + 'bbb@0-0', // existing site, earlier CGO, not applied + 'bbb@1-0', // existing site, same CGO, not applied + 'bbb@2-0', // existing site, later CGO, applied + 'aaa@0-0', // different site, earlier CGO, applied + 'ccc@9-0', // different site, later CGO, applied + ].entries()) { + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial, + state: [liveObjectsHelper.counterCreateOp({ objectId: counterIds[i], count: 10 })], + }); + } + + // check only operations with correct timeserials were applied + const expectedCounterValues = [ + 1, + 1, + 11, // applied COUNTER_CREATE + 11, // applied COUNTER_CREATE + 11, // applied COUNTER_CREATE + ]; + + for (const [i, counterId] of counterIds.entries()) { + const expectedValue = expectedCounterValues[i]; + + expect(root.get(counterId).value()).to.equal( + expectedValue, + `Check counter #${i + 1} has expected value after COUNTER_CREATE ops`, + ); + } + }, + }, + + { + description: 'can apply COUNTER_INC state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; const counterKey = 'counter'; @@ -894,24 +1175,67 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], } }, }, + + { + description: + 'COUNTER_INC state operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + // create new counter and set it on a root with forged timeserials + const counterId = liveObjectsHelper.fakeCounterObjectId(); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: 'bbb@1-0', + state: [liveObjectsHelper.counterCreateOp({ objectId: counterId, count: 1 })], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: 'aaa@0-0', + state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'counter', data: { objectId: counterId } })], + }); + + // inject operations with various timeserial values + for (const [i, serial] of [ + 'bbb@0-0', // +10 existing site, earlier CGO, not applied + 'bbb@1-0', // +100 existing site, same CGO, not applied + 'bbb@2-0', // +1000 existing site, later CGO, applied, site timeserials updated + 'bbb@2-0', // +10000 existing site, same CGO (updated from last op), not applied + 'aaa@0-0', // +100000 different site, earlier CGO, applied + 'ccc@9-0', // +1000000 different site, later CGO, applied + ].entries()) { + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial, + state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: Math.pow(10, i + 1) })], + }); + } + + // check only operations with correct timeserials were applied + expect(root.get('counter').value()).to.equal( + 1 + 1000 + 100000 + 1000000, // sum of passing operations and the initial value + `Check counter has expected value after COUNTER_INC ops`, + ); + }, + }, ]; forScenarios(applyOperationsScenarios, (scenario) => /** @nospec */ - it(`can apply ${scenario.description} state operation messages`, async function () { + it(scenario.description, async function () { const helper = this.test.helper; const liveObjectsHelper = new LiveObjectsHelper(helper); const client = RealtimeWithLiveObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channelName = `channel_can_apply_${scenario.description}`; + const channelName = scenario.description; const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); const liveObjects = channel.liveObjects; await channel.attach(); const root = await liveObjects.getRoot(); - await scenario.action({ root, liveObjectsHelper, channelName }); + await scenario.action({ root, liveObjectsHelper, channelName, channel }); }, client); }), ); @@ -960,10 +1284,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // inject operations, they should be applied when sync ends await Promise.all( - primitiveKeyData.map((keyData) => + primitiveKeyData.map((keyData, i) => liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: '@0-0', + serial: `aaa@${i}-0`, state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], }), ), @@ -1050,7 +1374,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { - description: 'buffered state operation messages are applied based on regional timeserial of the object', + description: + 'buffered state operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { const { root, liveObjectsHelper, channel } = ctx; @@ -1060,64 +1385,81 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await liveObjectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:cursor', - // add state object messages with non-zero regional timeserials + // add state object messages with non-empty site timeserials state: [ + // next map and counter objects will be checked to have correct operations applied on them based on site timeserials liveObjectsHelper.mapObject({ - objectId: 'root', - regionalTimeserial: '@1-0', + objectId: mapId, + siteTimeserials: { + bbb: 'bbb@2-0', + ccc: 'ccc@5-0', + }, entries: { - map: { timeserial: '@0-0', data: { objectId: mapId } }, - counter: { timeserial: '@0-0', data: { objectId: counterId } }, + foo1: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo2: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo3: { timeserial: 'ccc@5-0', data: { value: 'bar' } }, + foo4: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo5: { timeserial: 'bbb@2-0', data: { value: 'bar' } }, + foo6: { timeserial: 'ccc@2-0', data: { value: 'bar' } }, + foo7: { timeserial: 'ccc@0-0', data: { value: 'bar' } }, + foo8: { timeserial: 'ccc@0-0', data: { value: 'bar' } }, }, }), - liveObjectsHelper.mapObject({ - objectId: mapId, - regionalTimeserial: '@1-0', - }), liveObjectsHelper.counterObject({ objectId: counterId, - regionalTimeserial: '@1-0', + siteTimeserials: { + bbb: 'bbb@1-0', + }, + count: 1, + }), + // add objects to the root so they're discoverable in the state tree + liveObjectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { '000': '000@0-0' }, + entries: { + map: { timeserial: '000@0-0', data: { objectId: mapId } }, + counter: { timeserial: '000@0-0', data: { objectId: counterId } }, + }, }), ], }); - // inject operations with older or equal regional timeserial, expect them not to be applied when sync ends - await Promise.all( - ['@0-0', '@1-0'].map(async (serial) => { - await Promise.all( - ['root', mapId].flatMap((objectId) => - primitiveKeyData.map((keyData) => - liveObjectsHelper.processStateOperationMessageOnChannel({ - channel, - serial, - state: [liveObjectsHelper.mapSetOp({ objectId, key: keyData.key, data: keyData.data })], - }), - ), - ), - ); - await liveObjectsHelper.processStateOperationMessageOnChannel({ - channel, - serial, - state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], - }); - }), - ); + // inject operations with various timeserial values + // Map: + for (const [i, serial] of [ + 'bbb@1-0', // existing site, earlier site CGO, not applied + 'bbb@2-0', // existing site, same site CGO, not applied + 'bbb@3-0', // existing site, later site CGO, earlier entry CGO, not applied but site timeserial updated + // message with later site CGO, same entry CGO case is not possible, as timeserial from entry would be set for the corresponding site code or be less than that + 'bbb@3-0', // existing site, same site CGO (updated from last op), later entry CGO, not applied + 'bbb@4-0', // existing site, later site CGO, later entry CGO, applied + 'aaa@1-0', // different site, earlier entry CGO, not applied but site timeserial updated + 'aaa@1-0', // different site, same site CGO (updated from last op), later entry CGO, not applied + // different site with matching entry CGO case is not possible, as matching entry timeserial means that that timeserial is in the site timeserials vector + 'ddd@1-0', // different site, later entry CGO, applied + ].entries()) { + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial, + state: [liveObjectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { value: 'baz' } })], + }); + } - // inject operations with greater regional timeserial, expect them to be applied when sync ends - await Promise.all( - ['root', mapId].map((objectId) => - liveObjectsHelper.processStateOperationMessageOnChannel({ - channel, - serial: '@2-0', - state: [liveObjectsHelper.mapSetOp({ objectId, key: 'foo', data: { value: 'bar' } })], - }), - ), - ); - await liveObjectsHelper.processStateOperationMessageOnChannel({ - channel, - serial: '@2-0', - state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], - }); + // Counter: + for (const [i, serial] of [ + 'bbb@0-0', // +10 existing site, earlier CGO, not applied + 'bbb@1-0', // +100 existing site, same CGO, not applied + 'bbb@2-0', // +1000 existing site, later CGO, applied, site timeserials updated + 'bbb@2-0', // +10000 existing site, same CGO (updated from last op), not applied + 'aaa@0-0', // +100000 different site, earlier CGO, applied + 'ccc@9-0', // +1000000 different site, later CGO, applied + ].entries()) { + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial, + state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: Math.pow(10, i + 1) })], + }); + } // end sync await liveObjectsHelper.processStateObjectMessageOnChannel({ @@ -1125,33 +1467,28 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], syncSerial: 'serial:', }); - // check operations with older or equal regional timeserial are not applied - // counter will be checked to match an expected value explicitly, so no need to check that it doesn't equal a sum of operations - primitiveKeyData.forEach((keyData) => { - expect( - root.get(keyData.key), - `Check "${keyData.key}" key doesn't exist on root when STATE_SYNC has ended`, - ).to.not.exist; - }); - primitiveKeyData.forEach((keyData) => { - expect( - root.get('map').get(keyData.key), - `Check "${keyData.key}" key doesn't exist on inner map when STATE_SYNC has ended`, - ).to.not.exist; + // check only operations with correct timeserials were applied + const expectedMapKeys = [ + { key: 'foo1', value: 'bar' }, + { key: 'foo2', value: 'bar' }, + { key: 'foo3', value: 'bar' }, + { key: 'foo4', value: 'bar' }, + { key: 'foo5', value: 'baz' }, // updated + { key: 'foo6', value: 'bar' }, + { key: 'foo7', value: 'bar' }, + { key: 'foo8', value: 'baz' }, // updated + ]; + + expectedMapKeys.forEach(({ key, value }) => { + expect(root.get('map').get(key)).to.equal( + value, + `Check "${key}" key on map has expected value after STATE_SYNC has ended`, + ); }); - // check operations with greater regional timeserial are applied - expect(root.get('foo')).to.equal( - 'bar', - 'Check only data from operations with greater regional timeserial exists on root after STATE_SYNC', - ); - expect(root.get('map').get('foo')).to.equal( - 'bar', - 'Check only data from operations with greater regional timeserial exists on inner map after STATE_SYNC', - ); expect(root.get('counter').value()).to.equal( - 1, - 'Check only increment operations with greater regional timeserial were applied to counter after STATE_SYNC', + 1 + 1000 + 100000 + 1000000, // sum of passing operations and the initial value + `Check counter has expected value after STATE_SYNC has ended`, ); }, }, @@ -1170,10 +1507,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // inject operations, they should be applied when sync ends await Promise.all( - primitiveKeyData.map((keyData) => + primitiveKeyData.map((keyData, i) => liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: '@0-0', + serial: `aaa@${i}-0`, state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], }), ), From 1a828a6a5720b14726f534c0854c83b0cac85349 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 22 Nov 2024 04:03:21 +0000 Subject: [PATCH 066/166] Fix StateMessage.decode not correctly decoding operation.map.entries --- src/plugins/liveobjects/statemessage.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index d3af8bd3d6..bf1cd948a1 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -164,8 +164,8 @@ export class StateMessage { } } - if (message.operation?.map) { - for (const entry of Object.values(message.operation.map)) { + if (message.operation?.map?.entries) { + for (const entry of Object.values(message.operation.map.entries)) { await decodeMapEntry(entry, inputContext, decodeDataFn); } } From 86fc932d6275ca209152bed5312421df59c6119c Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 22 Nov 2024 04:04:05 +0000 Subject: [PATCH 067/166] Refactor decoding/encoding in StateMessage --- src/plugins/liveobjects/statemessage.ts | 219 +++++++++++++----------- 1 file changed, 120 insertions(+), 99 deletions(-) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index bf1cd948a1..08bbb2fa18 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -144,41 +144,16 @@ export class StateMessage { ): Promise { // TODO: decide how to handle individual errors from decoding values. currently we throw first ever error we get - const decodeMapEntry = async ( - entry: StateMapEntry, - ctx: ChannelOptions, - decode: typeof decodeData, - ): Promise => { - const { data, encoding, error } = await decode(entry.data.value, entry.data.encoding, ctx); - entry.data.value = data; - entry.data.encoding = encoding ?? undefined; - - if (error) { - throw error; - } - }; - if (message.object?.map?.entries) { - for (const entry of Object.values(message.object.map.entries)) { - await decodeMapEntry(entry, inputContext, decodeDataFn); - } + await this._decodeMapEntries(message.object.map.entries, inputContext, decodeDataFn); } if (message.operation?.map?.entries) { - for (const entry of Object.values(message.operation.map.entries)) { - await decodeMapEntry(entry, inputContext, decodeDataFn); - } + await this._decodeMapEntries(message.operation.map.entries, inputContext, decodeDataFn); } - if (message.operation?.mapOp?.data && 'value' in message.operation?.mapOp?.data) { - const mapOpData = message.operation.mapOp.data; - const { data, encoding, error } = await decodeDataFn(mapOpData.value, mapOpData.encoding, inputContext); - mapOpData.value = data; - mapOpData.encoding = encoding ?? undefined; - - if (error) { - throw error; - } + if (message.operation?.mapOp?.data && 'value' in message.operation.mapOp.data) { + await this._decodeStateData(message.operation.mapOp.data, inputContext, decodeDataFn); } } @@ -197,6 +172,114 @@ export class StateMessage { return result; } + private static async _decodeMapEntries( + mapEntries: Record, + inputContext: ChannelOptions, + decodeDataFn: typeof decodeData, + ): Promise { + for (const entry of Object.values(mapEntries)) { + await this._decodeStateData(entry.data, inputContext, decodeDataFn); + } + } + + private static async _decodeStateData( + stateData: StateData, + inputContext: ChannelOptions, + decodeDataFn: typeof decodeData, + ): Promise { + const { data, encoding, error } = await decodeDataFn(stateData.value, stateData.encoding, inputContext); + stateData.value = data; + stateData.encoding = encoding ?? undefined; + + if (error) { + throw error; + } + } + + private static _encodeStateOperation( + platform: typeof Platform, + stateOperation: StateOperation, + withBase64Encoding: boolean, + ): StateOperation { + // deep copy "stateOperation" object so we can modify the copy here. + // buffer values won't be correctly copied, so we will need to set them again explictly. + const stateOperationCopy = JSON.parse(JSON.stringify(stateOperation)) as StateOperation; + + if (stateOperationCopy.mapOp?.data && 'value' in stateOperationCopy.mapOp.data) { + // use original "stateOperation" object when encoding values, so we have access to the original buffer values. + stateOperationCopy.mapOp.data = this._encodeStateData(platform, stateOperation.mapOp?.data!, withBase64Encoding); + } + + if (stateOperationCopy.map?.entries) { + Object.entries(stateOperationCopy.map.entries).forEach(([key, entry]) => { + // use original "stateOperation" object when encoding values, so we have access to original buffer values. + entry.data = this._encodeStateData(platform, stateOperation?.map?.entries?.[key].data!, withBase64Encoding); + }); + } + + return stateOperationCopy; + } + + private static _encodeStateObject( + platform: typeof Platform, + stateObject: StateObject, + withBase64Encoding: boolean, + ): StateObject { + // deep copy "stateObject" object so we can modify the copy here. + // buffer values won't be correctly copied, so we will need to set them again explictly. + const stateObjectCopy = JSON.parse(JSON.stringify(stateObject)) as StateObject; + + if (stateObjectCopy.map?.entries) { + Object.entries(stateObjectCopy.map.entries).forEach(([key, entry]) => { + // use original "stateObject" object when encoding values, so we have access to original buffer values. + entry.data = StateMessage._encodeStateData( + platform, + stateObject?.map?.entries?.[key].data!, + withBase64Encoding, + ); + }); + } + + return stateObjectCopy; + } + + private static _encodeStateData(platform: typeof Platform, data: StateData, withBase64Encoding: boolean): StateData { + const { value, encoding } = this._encodeStateValue(platform, data?.value, data?.encoding, withBase64Encoding); + return { + ...data, + value, + encoding, + }; + } + + private static _encodeStateValue( + platform: typeof Platform, + value: StateValue | undefined, + encoding: string | undefined, + withBase64Encoding: boolean, + ): { + value: StateValue | undefined; + encoding: string | undefined; + } { + if (!value || !platform.BufferUtils.isBuffer(value)) { + return { value, encoding }; + } + + if (withBase64Encoding) { + return { + value: platform.BufferUtils.base64Encode(value), + encoding: encoding ? encoding + '/base64' : 'base64', + }; + } + + // toBuffer returns a datatype understandable by + // that platform's msgpack implementation (Buffer in node, Uint8Array in browsers) + return { + value: platform.BufferUtils.toBuffer(value), + encoding, + }; + } + /** * Overload toJSON() to intercept JSON.stringify() * @return {*} @@ -215,44 +298,18 @@ export class StateMessage { // if withBase64Encoding = false - we were called by msgpack const withBase64Encoding = arguments.length > 0; - let operationCopy: StateOperation | undefined = undefined; - if (this.operation) { - // deep copy "operation" prop so we can modify it here. - // buffer values won't be correctly copied, so we will need to set them again explictly - operationCopy = JSON.parse(JSON.stringify(this.operation)) as StateOperation; - - if (operationCopy.mapOp?.data && 'value' in operationCopy.mapOp.data) { - // use original "operation" prop when encoding values, so we have access to original buffer values. - operationCopy.mapOp.data = this._encodeStateData(this.operation.mapOp?.data!, withBase64Encoding); - } - - if (operationCopy.map?.entries) { - Object.entries(operationCopy.map.entries).forEach(([key, entry]) => { - // use original "operation" prop when encoding values, so we have access to original buffer values. - entry.data = this._encodeStateData(this.operation?.map?.entries?.[key].data!, withBase64Encoding); - }); - } - } - - let object: StateObject | undefined = undefined; - if (this.object) { - // deep copy "object" prop so we can modify it here. - // buffer values won't be correctly copied, so we will need to set them again explictly - object = JSON.parse(JSON.stringify(this.object)) as StateObject; - - if (object.map?.entries) { - Object.entries(object.map.entries).forEach(([key, entry]) => { - // use original "object" prop when encoding values, so we have access to original buffer values. - entry.data = this._encodeStateData(this.object?.map?.entries?.[key].data!, withBase64Encoding); - }); - } - } + const encodedOperation = this.operation + ? StateMessage._encodeStateOperation(this._platform, this.operation, withBase64Encoding) + : undefined; + const encodedObject = this.object + ? StateMessage._encodeStateObject(this._platform, this.object, withBase64Encoding) + : undefined; return { id: this.id, clientId: this.clientId, - operation: operationCopy, - object: object, + operation: encodedOperation, + object: encodedObject, extras: this.extras, }; } @@ -275,40 +332,4 @@ export class StateMessage { return result; } - - private _encodeStateData(data: StateData, withBase64Encoding: boolean): StateData { - const { value, encoding } = this._encodeStateValue(data?.value, data?.encoding, withBase64Encoding); - return { - ...data, - value, - encoding, - }; - } - - private _encodeStateValue( - value: StateValue | undefined, - encoding: string | undefined, - withBase64Encoding: boolean, - ): { - value: StateValue | undefined; - encoding: string | undefined; - } { - if (!value || !this._platform.BufferUtils.isBuffer(value)) { - return { value, encoding }; - } - - if (withBase64Encoding) { - return { - value: this._platform.BufferUtils.base64Encode(value), - encoding: encoding ? encoding + '/base64' : 'base64', - }; - } - - // toBuffer returns a datatype understandable by - // that platform's msgpack implementation (Buffer in node, Uint8Array in browsers) - return { - value: this._platform.BufferUtils.toBuffer(value), - encoding, - }; - } } From 54d4555bd5ae96634264813b387029f62859aff5 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 22 Nov 2024 08:10:32 +0000 Subject: [PATCH 068/166] Handle `createOp` field on StateObject messages Refactors LiveMap/LiveCounter object creation and moves most of the creation related busy work inside those classes. Resolves DTP-1076 --- src/plugins/liveobjects/livecounter.ts | 128 ++++++----- src/plugins/liveobjects/livemap.ts | 204 ++++++++++++------ src/plugins/liveobjects/liveobject.ts | 83 ++++--- src/plugins/liveobjects/liveobjects.ts | 19 +- src/plugins/liveobjects/liveobjectspool.ts | 75 +------ src/plugins/liveobjects/statemessage.ts | 35 ++- .../liveobjects/syncliveobjectsdatapool.ts | 41 +--- test/common/modules/live_objects_helper.js | 28 ++- test/realtime/live_objects.test.js | 8 +- 9 files changed, 347 insertions(+), 274 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 0327adafcb..bfa3a99cce 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -1,7 +1,7 @@ import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; -import { StateCounter, StateCounterOp, StateMessage, StateOperation, StateOperationAction } from './statemessage'; -import { DefaultTimeserial, Timeserial } from './timeserial'; +import { StateCounterOp, StateMessage, StateObject, StateOperation, StateOperationAction } from './statemessage'; +import { DefaultTimeserial } from './timeserial'; export interface LiveCounterData extends LiveObjectData { data: number; @@ -12,46 +12,29 @@ export interface LiveCounterUpdate extends LiveObjectUpdate { } export class LiveCounter extends LiveObject { - constructor( - liveObjects: LiveObjects, - private _created: boolean, - initialData?: LiveCounterData | null, - objectId?: string, - siteTimeserials?: Record, - ) { - super(liveObjects, initialData, objectId, siteTimeserials); - } - /** * Returns a {@link LiveCounter} instance with a 0 value. * * @internal */ - static zeroValue( - liveobjects: LiveObjects, - isCreated: boolean, - objectId?: string, - siteTimeserials?: Record, - ): LiveCounter { - return new LiveCounter(liveobjects, isCreated, null, objectId, siteTimeserials); - } - - value(): number { - return this._dataRef.data; + static zeroValue(liveobjects: LiveObjects, objectId: string): LiveCounter { + return new LiveCounter(liveobjects, objectId); } /** + * Returns a {@link LiveCounter} instance based on the provided state object. + * The provided state object must hold a valid counter object data. + * * @internal */ - isCreated(): boolean { - return this._created; + static fromStateObject(liveobjects: LiveObjects, stateObject: StateObject): LiveCounter { + const obj = new LiveCounter(liveobjects, stateObject.objectId); + obj.overrideWithStateObject(stateObject); + return obj; } - /** - * @internal - */ - setCreated(created: boolean): void { - this._created = created; + value(): number { + return this._dataRef.data; } /** @@ -83,7 +66,7 @@ export class LiveCounter extends LiveObject let update: LiveCounterUpdate | LiveObjectUpdateNoop; switch (op.action) { case StateOperationAction.COUNTER_CREATE: - update = this._applyCounterCreate(op.counter); + update = this._applyCounterCreate(op); break; case StateOperationAction.COUNTER_INC: @@ -107,15 +90,69 @@ export class LiveCounter extends LiveObject this.notifyUpdated(update); } + /** + * @internal + */ + overrideWithStateObject(stateObject: StateObject): LiveCounterUpdate { + if (stateObject.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Invalid state object: state object objectId=${stateObject.objectId}; LiveCounter objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + + if (!this._client.Utils.isNil(stateObject.createOp)) { + // it is expected that create operation can be missing in the state object, so only validate it when it exists + if (stateObject.createOp.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Invalid state object: state object createOp objectId=${stateObject.createOp?.objectId}; LiveCounter objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + + if (stateObject.createOp.action !== StateOperationAction.COUNTER_CREATE) { + throw new this._client.ErrorInfo( + `Invalid state object: state object createOp action=${stateObject.createOp?.action}; LiveCounter objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + } + + const previousDataRef = this._dataRef; + // override all relevant data for this object with data from the state object + this._createOperationIsMerged = false; + this._dataRef = { data: stateObject.counter?.count ?? 0 }; + this._siteTimeserials = this._timeserialMapFromStringMap(stateObject.siteTimeserials); + if (!this._client.Utils.isNil(stateObject.createOp)) { + this._mergeInitialDataFromCreateOperation(stateObject.createOp); + } + + return this._updateFromDataDiff(previousDataRef, this._dataRef); + } + protected _getZeroValueData(): LiveCounterData { return { data: 0 }; } - protected _updateFromDataDiff(currentDataRef: LiveCounterData, newDataRef: LiveCounterData): LiveCounterUpdate { - const counterDiff = newDataRef.data - currentDataRef.data; + protected _updateFromDataDiff(prevDataRef: LiveCounterData, newDataRef: LiveCounterData): LiveCounterUpdate { + const counterDiff = newDataRef.data - prevDataRef.data; return { update: { inc: counterDiff } }; } + protected _mergeInitialDataFromCreateOperation(stateOperation: StateOperation): LiveCounterUpdate { + // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly 0 in this case. + // note that it is intentional to SUM the incoming count from the create op. + // if we got here, it means that current counter instance is missing the initial value in its data reference, + // which we're going to add now. + this._dataRef.data += stateOperation.counter?.count ?? 0; + this._createOperationIsMerged = true; + + return { update: { inc: stateOperation.counter?.count ?? 0 } }; + } + private _throwNoPayloadError(op: StateOperation): void { throw new this._client.ErrorInfo( `No payload found for ${op.action} op for LiveCounter objectId=${this.getObjectId()}`, @@ -124,32 +161,21 @@ export class LiveCounter extends LiveObject ); } - private _applyCounterCreate(op: StateCounter | undefined): LiveCounterUpdate | LiveObjectUpdateNoop { - if (this.isCreated()) { - // skip COUNTER_CREATE op if this counter is already created + private _applyCounterCreate(op: StateOperation): LiveCounterUpdate | LiveObjectUpdateNoop { + if (this._createOperationIsMerged) { + // There can't be two different create operation for the same object id, because the object id + // fully encodes that operation. This means we can safely ignore any new incoming create operations + // if we already merged it once. this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MICRO, 'LiveCounter._applyCounterCreate()', - `skipping applying COUNTER_CREATE op on a counter instance as it is already created; objectId=${this._objectId}`, + `skipping applying COUNTER_CREATE op on a counter instance as it was already applied before; objectId=${this._objectId}`, ); return { noop: true }; } - if (this._client.Utils.isNil(op)) { - // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly 0 in this case. - // we need to SUM the initial value to the current value due to the reasons below, but since it's a 0, we can skip addition operation - this.setCreated(true); - return { update: { inc: 0 } }; - } - - // note that it is intentional to SUM the incoming count from the create op. - // if we get here, it means that current counter instance wasn't initialized from the COUNTER_CREATE op, - // so it is missing the initial value that we're going to add now. - this._dataRef.data += op.count ?? 0; - this.setCreated(true); - - return { update: { inc: op.count ?? 0 } }; + return this._mergeInitialDataFromCreateOperation(op); } private _applyCounterInc(op: StateCounterOp): LiveCounterUpdate { diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index de19baa961..b29654c02f 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -1,14 +1,13 @@ import deepEqual from 'deep-equal'; -import type BaseClient from 'common/lib/client/baseclient'; import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; import { MapSemantics, - StateMap, StateMapEntry, StateMapOp, StateMessage, + StateObject, StateOperation, StateOperationAction, StateValue, @@ -50,11 +49,9 @@ export class LiveMap extends LiveObject { constructor( liveObjects: LiveObjects, private _semantics: MapSemantics, - initialData?: LiveMapData | null, - objectId?: string, - siteTimeserials?: Record, + objectId: string, ) { - super(liveObjects, initialData, objectId, siteTimeserials); + super(liveObjects, objectId); } /** @@ -62,41 +59,20 @@ export class LiveMap extends LiveObject { * * @internal */ - static zeroValue(liveobjects: LiveObjects, objectId?: string, siteTimeserials?: Record): LiveMap { - return new LiveMap(liveobjects, MapSemantics.LWW, null, objectId, siteTimeserials); + static zeroValue(liveobjects: LiveObjects, objectId: string): LiveMap { + return new LiveMap(liveobjects, MapSemantics.LWW, objectId); } /** + * Returns a {@link LiveMap} instance based on the provided state object. + * The provided state object must hold a valid map object data. + * * @internal */ - static liveMapDataFromMapEntries(client: BaseClient, entries: Record): LiveMapData { - const liveMapData: LiveMapData = { - data: new Map(), - }; - - // need to iterate over entries manually to work around optional parameters from state object entries type - Object.entries(entries ?? {}).forEach(([key, entry]) => { - let liveData: StateData; - if (typeof entry.data.objectId !== 'undefined') { - liveData = { objectId: entry.data.objectId } as ObjectIdStateData; - } else { - liveData = { encoding: entry.data.encoding, value: entry.data.value } as ValueStateData; - } - - const liveDataEntry: MapEntry = { - ...entry, - timeserial: entry.timeserial - ? DefaultTimeserial.calculateTimeserial(client, entry.timeserial) - : DefaultTimeserial.zeroValueTimeserial(client), - // true only if we received explicit true. otherwise always false - tombstone: entry.tombstone === true, - data: liveData, - }; - - liveMapData.data.set(key, liveDataEntry); - }); - - return liveMapData; + static fromStateObject(liveobjects: LiveObjects, stateObject: StateObject): LiveMap { + const obj = new LiveMap(liveobjects, stateObject.map?.semantics!, stateObject.objectId); + obj.overrideWithStateObject(stateObject); + return obj; } /** @@ -170,7 +146,7 @@ export class LiveMap extends LiveObject { let update: LiveMapUpdate | LiveObjectUpdateNoop; switch (op.action) { case StateOperationAction.MAP_CREATE: - update = this._applyMapCreate(op.map); + update = this._applyMapCreate(op); break; case StateOperationAction.MAP_SET: @@ -204,14 +180,73 @@ export class LiveMap extends LiveObject { this.notifyUpdated(update); } + /** + * @internal + */ + overrideWithStateObject(stateObject: StateObject): LiveMapUpdate { + if (stateObject.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Invalid state object: state object objectId=${stateObject.objectId}; LiveMap objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + + if (stateObject.map?.semantics !== this._semantics) { + throw new this._client.ErrorInfo( + `Invalid state object: state object map semantics=${stateObject.map?.semantics}; LiveMap semantics=${this._semantics}`, + 50000, + 500, + ); + } + + if (!this._client.Utils.isNil(stateObject.createOp)) { + // it is expected that create operation can be missing in the state object, so only validate it when it exists + if (stateObject.createOp.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Invalid state object: state object createOp objectId=${stateObject.createOp?.objectId}; LiveMap objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + + if (stateObject.createOp.action !== StateOperationAction.MAP_CREATE) { + throw new this._client.ErrorInfo( + `Invalid state object: state object createOp action=${stateObject.createOp?.action}; LiveMap objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + + if (stateObject.createOp.map?.semantics !== this._semantics) { + throw new this._client.ErrorInfo( + `Invalid state object: state object createOp map semantics=${stateObject.createOp.map?.semantics}; LiveMap semantics=${this._semantics}`, + 50000, + 500, + ); + } + } + + const previousDataRef = this._dataRef; + // override all relevant data for this object with data from the state object + this._createOperationIsMerged = false; + this._dataRef = this._liveMapDataFromMapEntries(stateObject.map?.entries ?? {}); + this._siteTimeserials = this._timeserialMapFromStringMap(stateObject.siteTimeserials); + if (!this._client.Utils.isNil(stateObject.createOp)) { + this._mergeInitialDataFromCreateOperation(stateObject.createOp); + } + + return this._updateFromDataDiff(previousDataRef, this._dataRef); + } + protected _getZeroValueData(): LiveMapData { return { data: new Map() }; } - protected _updateFromDataDiff(currentDataRef: LiveMapData, newDataRef: LiveMapData): LiveMapUpdate { + protected _updateFromDataDiff(prevDataRef: LiveMapData, newDataRef: LiveMapData): LiveMapUpdate { const update: LiveMapUpdate = { update: {} }; - for (const [key, currentEntry] of currentDataRef.data.entries()) { + for (const [key, currentEntry] of prevDataRef.data.entries()) { // any non-tombstoned properties that exist on a current map, but not in the new data - got removed if (currentEntry.tombstone === false && !newDataRef.data.has(key)) { update.update[key] = 'removed'; @@ -219,7 +254,7 @@ export class LiveMap extends LiveObject { } for (const [key, newEntry] of newDataRef.data.entries()) { - if (!currentDataRef.data.has(key)) { + if (!prevDataRef.data.has(key)) { // if property does not exist in the current map, but new data has it as a non-tombstoned property - got updated if (newEntry.tombstone === false) { update.update[key] = 'updated'; @@ -233,7 +268,7 @@ export class LiveMap extends LiveObject { } // properties that exist both in current and new map data need to have their values compared to decide on the update type - const currentEntry = currentDataRef.data.get(key)!; + const currentEntry = prevDataRef.data.get(key)!; // compare tombstones first if (currentEntry.tombstone === true && newEntry.tombstone === false) { @@ -262,33 +297,17 @@ export class LiveMap extends LiveObject { return update; } - private _throwNoPayloadError(op: StateOperation): void { - throw new this._client.ErrorInfo( - `No payload found for ${op.action} op for LiveMap objectId=${this.getObjectId()}`, - 50000, - 500, - ); - } - - private _applyMapCreate(op: StateMap | undefined): LiveMapUpdate | LiveObjectUpdateNoop { - if (this._client.Utils.isNil(op)) { + protected _mergeInitialDataFromCreateOperation(stateOperation: StateOperation): LiveMapUpdate { + if (this._client.Utils.isNil(stateOperation.map)) { // if a map object is missing for the MAP_CREATE op, the initial value is implicitly an empty map. // in this case there is nothing to merge into the current map, so we can just end processing the op. return { update: {} }; } - if (this._semantics !== op.semantics) { - throw new this._client.ErrorInfo( - `Cannot apply MAP_CREATE op on LiveMap objectId=${this.getObjectId()}; map's semantics=${this._semantics}, but op expected ${op.semantics}`, - 50000, - 500, - ); - } - const aggregatedUpdate: LiveMapUpdate | LiveObjectUpdateNoop = { update: {} }; // in order to apply MAP_CREATE op for an existing map, we should merge their underlying entries keys. // we can do this by iterating over entries from MAP_CREATE op and apply changes on per-key basis as if we had MAP_SET, MAP_REMOVE operations. - Object.entries(op.entries ?? {}).forEach(([key, entry]) => { + Object.entries(stateOperation.map.entries ?? {}).forEach(([key, entry]) => { // for MAP_CREATE op we must use dedicated timeserial field available on an entry, instead of a timeserial on a message const opOriginTimeserial = entry.timeserial ? DefaultTimeserial.calculateTimeserial(this._client, entry.timeserial) @@ -311,9 +330,44 @@ export class LiveMap extends LiveObject { Object.assign(aggregatedUpdate.update, update.update); }); + this._createOperationIsMerged = true; + return aggregatedUpdate; } + private _throwNoPayloadError(op: StateOperation): void { + throw new this._client.ErrorInfo( + `No payload found for ${op.action} op for LiveMap objectId=${this.getObjectId()}`, + 50000, + 500, + ); + } + + private _applyMapCreate(op: StateOperation): LiveMapUpdate | LiveObjectUpdateNoop { + if (this._createOperationIsMerged) { + // There can't be two different create operation for the same object id, because the object id + // fully encodes that operation. This means we can safely ignore any new incoming create operations + // if we already merged it once. + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveMap._applyMapCreate()', + `skipping applying MAP_CREATE op on a map instance as it was already applied before; objectId=${this._objectId}`, + ); + return { noop: true }; + } + + if (this._semantics !== op.map?.semantics) { + throw new this._client.ErrorInfo( + `Cannot apply MAP_CREATE op on LiveMap objectId=${this.getObjectId()}; map's semantics=${this._semantics}, but op expected ${op.map?.semantics}`, + 50000, + 500, + ); + } + + return this._mergeInitialDataFromCreateOperation(op); + } + private _applyMapSet(op: StateMapOp, opOriginTimeserial: Timeserial): LiveMapUpdate | LiveObjectUpdateNoop { const { ErrorInfo, Utils } = this._client; @@ -398,4 +452,34 @@ export class LiveMap extends LiveObject { return { update: { [op.key]: 'removed' } }; } + + private _liveMapDataFromMapEntries(entries: Record): LiveMapData { + const liveMapData: LiveMapData = { + data: new Map(), + }; + + // need to iterate over entries manually to work around optional parameters from state object entries type + Object.entries(entries ?? {}).forEach(([key, entry]) => { + let liveData: StateData; + if (typeof entry.data.objectId !== 'undefined') { + liveData = { objectId: entry.data.objectId } as ObjectIdStateData; + } else { + liveData = { encoding: entry.data.encoding, value: entry.data.value } as ValueStateData; + } + + const liveDataEntry: MapEntry = { + ...entry, + timeserial: entry.timeserial + ? DefaultTimeserial.calculateTimeserial(this._client, entry.timeserial) + : DefaultTimeserial.zeroValueTimeserial(this._client), + // true only if we received explicit true. otherwise always false + tombstone: entry.tombstone === true, + data: liveData, + }; + + liveMapData.data.set(key, liveDataEntry); + }); + + return liveMapData; + } } diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index e18762dbd8..04ae7d4e24 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -1,8 +1,8 @@ import type BaseClient from 'common/lib/client/baseclient'; import type EventEmitter from 'common/lib/util/eventemitter'; import { LiveObjects } from './liveobjects'; -import { StateMessage, StateOperation } from './statemessage'; -import { Timeserial } from './timeserial'; +import { StateMessage, StateObject, StateOperation } from './statemessage'; +import { DefaultTimeserial, Timeserial } from './timeserial'; enum LiveObjectEvents { Updated = 'Updated', @@ -32,22 +32,26 @@ export abstract class LiveObject< > { protected _client: BaseClient; protected _eventEmitter: EventEmitter; - protected _dataRef: TData; protected _objectId: string; + /** + * Represents an aggregated value for an object, which combines the initial value for an object from the create operation, + * and all state operations applied to the object. + */ + protected _dataRef: TData; protected _siteTimeserials: Record; + protected _createOperationIsMerged: boolean; - constructor( + protected constructor( protected _liveObjects: LiveObjects, - initialData?: TData | null, - objectId?: string, - siteTimeserials?: Record, + objectId: string, ) { this._client = this._liveObjects.getClient(); this._eventEmitter = new this._client.EventEmitter(this._client.logger); - this._dataRef = initialData ?? this._getZeroValueData(); - this._objectId = objectId ?? this._createObjectId(); + this._dataRef = this._getZeroValueData(); + this._createOperationIsMerged = false; + this._objectId = objectId; // use empty timeserials vector by default, so any future operation can be applied to this object - this._siteTimeserials = siteTimeserials ?? {}; + this._siteTimeserials = {}; } subscribe(listener: (update: TUpdate) => void): SubscribeResponse { @@ -83,26 +87,10 @@ export abstract class LiveObject< } /** - * Sets a new data reference for the LiveObject and returns an update object that describes the changes applied based on the object's previous value. + * Emits the {@link LiveObjectEvents.Updated} event with provided update object if it isn't a noop. * * @internal */ - setData(newDataRef: TData): TUpdate { - const update = this._updateFromDataDiff(this._dataRef, newDataRef); - this._dataRef = newDataRef; - return update; - } - - /** - * @internal - */ - setSiteTimeserials(siteTimeserials: Record): void { - this._siteTimeserials = siteTimeserials; - } - - /** - * @internal - */ notifyUpdated(update: TUpdate | LiveObjectUpdateNoop): void { // should not emit update event if update was noop if ((update as LiveObjectUpdateNoop).noop) { @@ -123,18 +111,57 @@ export abstract class LiveObject< return !siteTimeserial || opOriginTimeserial.after(siteTimeserial); } + protected _timeserialMapFromStringMap(stringTimeserialsMap: Record): Record { + const objTimeserialsMap = Object.entries(stringTimeserialsMap).reduce( + (acc, v) => { + const [key, timeserialString] = v; + acc[key] = DefaultTimeserial.calculateTimeserial(this._client, timeserialString); + return acc; + }, + {} as Record, + ); + + return objTimeserialsMap; + } + private _createObjectId(): string { // TODO: implement object id generation based on live object type and initial value return Math.random().toString().substring(2); } /** + * Apply state operation message on live object. + * * @internal */ abstract applyOperation(op: StateOperation, msg: StateMessage): void; + /** + * Overrides internal data for live object with data from the given state object. + * Provided state object should hold a valid data for current live object, e.g. counter data for LiveCounter, map data for LiveMap. + * + * State objects are received during SYNC sequence, and SYNC sequence is a source of truth for the current state of the objects, + * so we can use the data received from the SYNC sequence directly and override any data values or site timeserials this live object has + * without the need to merge them. + * + * Returns an update object that describes the changes applied based on the object's previous value. + * + * @internal + */ + abstract overrideWithStateObject(stateObject: StateObject): TUpdate; protected abstract _getZeroValueData(): TData; /** * Calculate the update object based on the current Live Object data and incoming new data. */ - protected abstract _updateFromDataDiff(currentDataRef: TData, newDataRef: TData): TUpdate; + protected abstract _updateFromDataDiff(prevDataRef: TData, newDataRef: TData): TUpdate; + /** + * Merges the initial data from the create operation into the live object state. + * + * Client SDKs do not need to keep around the state operation that created the object, + * so we can merge the initial data the first time we receive it for the object, + * and work with aggregated value after that. + * + * This saves us from needing to merge the initial value with operations applied to + * the object every time the object is read. + */ + protected abstract _mergeInitialDataFromCreateOperation(stateOperation: StateOperation): TUpdate; } diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 9791498e0c..5ba586fe42 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -7,7 +7,7 @@ import { LiveMap } from './livemap'; import { LiveObject, LiveObjectUpdate } from './liveobject'; import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage } from './statemessage'; -import { LiveCounterDataEntry, SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; +import { SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; enum LiveObjectsEvents { SyncCompleted = 'SyncCompleted', @@ -195,16 +195,9 @@ export class LiveObjects { const existingObject = this._liveObjectsPool.get(objectId); if (existingObject) { - // SYNC sequence is a source of truth for the current state of the objects, - // so we can use the data received from the SYNC sequence directly - // without the need to merge data values or site timeserials. - const update = existingObject.setData(entry.objectData); - existingObject.setSiteTimeserials(entry.siteTimeserials); - if (existingObject instanceof LiveCounter) { - existingObject.setCreated((entry as LiveCounterDataEntry).created); - } - // store updates for existing objects to call subscription callbacks for all of them once the SYNC sequence is completed. - // this will ensure that clients get notified about changes only once everything was applied. + const update = existingObject.overrideWithStateObject(entry.stateObject); + // store updates to call subscription callbacks for all of them once the SYNC sequence is completed. + // this will ensure that clients get notified about the changes only once everything has been applied. existingObjectUpdates.push({ object: existingObject, update }); continue; } @@ -214,11 +207,11 @@ export class LiveObjects { const objectType = entry.objectType; switch (objectType) { case 'LiveCounter': - newObject = new LiveCounter(this, entry.created, entry.objectData, objectId, entry.siteTimeserials); + newObject = LiveCounter.fromStateObject(this, entry.stateObject); break; case 'LiveMap': - newObject = new LiveMap(this, entry.semantics, entry.objectData, objectId, entry.siteTimeserials); + newObject = LiveMap.fromStateObject(this, entry.stateObject); break; default: diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index 22f1edb943..2c57f1084e 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -5,8 +5,7 @@ import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; import { LiveObjects } from './liveobjects'; import { ObjectId } from './objectid'; -import { MapSemantics, StateMessage, StateOperation, StateOperationAction } from './statemessage'; -import { DefaultTimeserial, Timeserial } from './timeserial'; +import { StateMessage, StateOperationAction } from './statemessage'; export const ROOT_OBJECT_ID = 'root'; @@ -60,7 +59,7 @@ export class LiveObjectsPool { } case 'counter': - zeroValueObject = LiveCounter.zeroValue(this._liveObjects, false, objectId); + zeroValueObject = LiveCounter.zeroValue(this._liveObjects, objectId); break; } @@ -79,35 +78,20 @@ export class LiveObjectsPool { continue; } - const opOriginTimeserial = DefaultTimeserial.calculateTimeserial(this._client, stateMessage.serial); const stateOperation = stateMessage.operation; switch (stateOperation.action) { case StateOperationAction.MAP_CREATE: case StateOperationAction.COUNTER_CREATE: - if (this.get(stateOperation.objectId)) { - // object wich such id already exists (we may have created a zero-value object before, or this is a duplicate *_CREATE op), - // so delegate application of the op to that object - this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); - break; - } - - // otherwise we can create new objects in the pool - if (stateOperation.action === StateOperationAction.MAP_CREATE) { - this._handleMapCreate(stateOperation, opOriginTimeserial); - } - - if (stateOperation.action === StateOperationAction.COUNTER_CREATE) { - this._handleCounterCreate(stateOperation, opOriginTimeserial); - } - break; - case StateOperationAction.MAP_SET: case StateOperationAction.MAP_REMOVE: case StateOperationAction.COUNTER_INC: // we can receive an op for an object id we don't have yet in the pool. instead of buffering such operations, - // we create a zero-value object for the provided object id, and apply operation for that zero-value object. - // when we eventually receive a corresponding *_CREATE op for that object, its application will be handled by that zero-value object. + // we can create a zero-value object for the provided object id and apply the operation to that zero-value object. + // this also means that all objects are capable of applying the corresponding *_CREATE ops on themselves, + // since they need to be able to eventually initialize themselves from that *_CREATE op. + // so to simplify operations handling, we always try to create a zero-value object in the pool first, + // and then we can always apply the operation on the existing object in the pool. this.createZeroValueObjectIfNotExists(stateOperation.objectId); this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); break; @@ -129,49 +113,4 @@ export class LiveObjectsPool { pool.set(root.getObjectId(), root); return pool; } - - private _handleCounterCreate(stateOperation: StateOperation, opOriginTimeserial: Timeserial): void { - // should use op's origin timeserial as the initial value for the object's site timeserials vector - const siteTimeserials = { - [opOriginTimeserial.siteCode]: opOriginTimeserial, - }; - let counter: LiveCounter; - if (this._client.Utils.isNil(stateOperation.counter)) { - // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly a zero-value counter. - counter = LiveCounter.zeroValue(this._liveObjects, true, stateOperation.objectId, siteTimeserials); - } else { - counter = new LiveCounter( - this._liveObjects, - true, - { data: stateOperation.counter.count ?? 0 }, - stateOperation.objectId, - siteTimeserials, - ); - } - - this.set(stateOperation.objectId, counter); - } - - private _handleMapCreate(stateOperation: StateOperation, opOriginTimeserial: Timeserial): void { - // should use op's origin timeserial as the initial value for the object's site timeserials vector - const siteTimeserials = { - [opOriginTimeserial.siteCode]: opOriginTimeserial, - }; - let map: LiveMap; - if (this._client.Utils.isNil(stateOperation.map)) { - // if a map object is missing for the MAP_CREATE op, the initial value is implicitly a zero-value map. - map = LiveMap.zeroValue(this._liveObjects, stateOperation.objectId, siteTimeserials); - } else { - const objectData = LiveMap.liveMapDataFromMapEntries(this._client, stateOperation.map.entries ?? {}); - map = new LiveMap( - this._liveObjects, - stateOperation.map.semantics ?? MapSemantics.LWW, - objectData, - stateOperation.objectId, - siteTimeserials, - ); - } - - this.set(stateOperation.objectId, map); - } } diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 08bbb2fa18..f271ef59e8 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -71,12 +71,6 @@ export interface StateMap { export interface StateCounter { /** The value of the counter */ count?: number; - /** - * Indicates (true) if the counter has seen an explicit create operation - * and false if the counter was created with a default value when - * processing a regular operation. - */ - created: boolean; } /** A StateOperation describes an operation to be applied to a state object. */ @@ -112,9 +106,21 @@ export interface StateObject { objectId: string; /** A vector of origin timeserials keyed by site code of the last operation that was applied to this state object. */ siteTimeserials: Record; - /** The data that represents the state of the object if it is a Map object type. */ + /** + * The operation that created the state object. + * + * Can be missing if create operation for the object is not known at this point. + */ + createOp?: StateOperation; + /** + * The data that represents the result of applying all operations to a Map object + * excluding the initial value from the create operation if it is a Map object type. + */ map?: StateMap; - /** The data that represents the state of the object if it is a Counter object type. */ + /** + * The data that represents the result of applying all operations to a Counter object + * excluding the initial value from the create operation if it is a Counter object type. + */ counter?: StateCounter; } @@ -148,6 +154,14 @@ export class StateMessage { await this._decodeMapEntries(message.object.map.entries, inputContext, decodeDataFn); } + if (message.object?.createOp?.map?.entries) { + await this._decodeMapEntries(message.object.createOp.map.entries, inputContext, decodeDataFn); + } + + if (message.object?.createOp?.mapOp?.data && 'value' in message.object.createOp.mapOp.data) { + await this._decodeStateData(message.object.createOp.mapOp.data, inputContext, decodeDataFn); + } + if (message.operation?.map?.entries) { await this._decodeMapEntries(message.operation.map.entries, inputContext, decodeDataFn); } @@ -240,6 +254,11 @@ export class StateMessage { }); } + if (stateObjectCopy.createOp) { + // use original "stateObject" object when encoding values, so we have access to original buffer values. + stateObjectCopy.createOp = this._encodeStateOperation(platform, stateObject.createOp!, withBase64Encoding); + } + return stateObjectCopy; } diff --git a/src/plugins/liveobjects/syncliveobjectsdatapool.ts b/src/plugins/liveobjects/syncliveobjectsdatapool.ts index 194c3165c0..663a1bb216 100644 --- a/src/plugins/liveobjects/syncliveobjectsdatapool.ts +++ b/src/plugins/liveobjects/syncliveobjectsdatapool.ts @@ -1,30 +1,24 @@ import type BaseClient from 'common/lib/client/baseclient'; import type RealtimeChannel from 'common/lib/client/realtimechannel'; -import { LiveCounterData } from './livecounter'; -import { LiveMap } from './livemap'; -import { LiveObjectData } from './liveobject'; import { LiveObjects } from './liveobjects'; -import { MapSemantics, StateMessage, StateObject } from './statemessage'; -import { DefaultTimeserial, Timeserial } from './timeserial'; +import { StateMessage, StateObject } from './statemessage'; export interface LiveObjectDataEntry { - objectData: LiveObjectData; - siteTimeserials: Record; + stateObject: StateObject; objectType: 'LiveMap' | 'LiveCounter'; } export interface LiveCounterDataEntry extends LiveObjectDataEntry { - created: boolean; objectType: 'LiveCounter'; } export interface LiveMapDataEntry extends LiveObjectDataEntry { objectType: 'LiveMap'; - semantics: MapSemantics; } export type AnyDataEntry = LiveCounterDataEntry | LiveMapDataEntry; +// TODO: investigate if this class is still needed after changes with createOp. objects are now initialized from the stateObject and this class does minimal processing /** * @internal */ @@ -85,45 +79,20 @@ export class SyncLiveObjectsDataPool { } private _createLiveCounterDataEntry(stateObject: StateObject): LiveCounterDataEntry { - const counter = stateObject.counter!; - - const objectData: LiveCounterData = { - data: counter.count ?? 0, - }; const newEntry: LiveCounterDataEntry = { - objectData, + stateObject, objectType: 'LiveCounter', - siteTimeserials: this._timeserialMapFromStringMap(stateObject.siteTimeserials), - created: counter.created, }; return newEntry; } private _createLiveMapDataEntry(stateObject: StateObject): LiveMapDataEntry { - const map = stateObject.map!; - const objectData = LiveMap.liveMapDataFromMapEntries(this._client, map.entries ?? {}); - const newEntry: LiveMapDataEntry = { - objectData, + stateObject, objectType: 'LiveMap', - siteTimeserials: this._timeserialMapFromStringMap(stateObject.siteTimeserials), - semantics: map.semantics ?? MapSemantics.LWW, }; return newEntry; } - - private _timeserialMapFromStringMap(stringTimeserialsMap: Record): Record { - const objTimeserialsMap = Object.entries(stringTimeserialsMap).reduce( - (acc, v) => { - const [key, timeserialString] = v; - acc[key] = DefaultTimeserial.calculateTimeserial(this._client, timeserialString); - return acc; - }, - {} as Record, - ); - - return objTimeserialsMap; - } } diff --git a/test/common/modules/live_objects_helper.js b/test/common/modules/live_objects_helper.js index 9f8e98783c..273a2f4999 100644 --- a/test/common/modules/live_objects_helper.js +++ b/test/common/modules/live_objects_helper.js @@ -101,11 +101,17 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb action: ACTIONS.MAP_CREATE, nonce: nonce(), objectId, + map: { + semantics: 0, + }, }, }; if (entries) { - op.operation.map = { entries }; + op.operation.map = { + ...op.operation.map, + entries, + }; } return op; @@ -175,31 +181,41 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb } mapObject(opts) { - const { objectId, siteTimeserials, entries } = opts; + const { objectId, siteTimeserials, initialEntries, materialisedEntries } = opts; const obj = { object: { objectId, siteTimeserials, - map: { entries }, + map: { + semantics: 0, + entries: materialisedEntries, + }, }, }; + if (initialEntries) { + obj.object.createOp = this.mapCreateOp({ objectId, entries: initialEntries }).operation; + } + return obj; } counterObject(opts) { - const { objectId, siteTimeserials, count } = opts; + const { objectId, siteTimeserials, initialCount, materialisedCount } = opts; const obj = { object: { objectId, siteTimeserials, counter: { - created: true, - count, + count: materialisedCount, }, }, }; + if (initialCount != null) { + obj.object.createOp = this.counterCreateOp({ objectId, count: initialCount }).operation; + } + return obj; } diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 34ded67714..ead316addc 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -293,7 +293,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], liveObjectsHelper.mapObject({ objectId: 'root', siteTimeserials: { '000': '000@0-0' }, - entries: { key: { timeserial: '000@0-0', data: { value: 1 } } }, + initialEntries: { key: { timeserial: '000@0-0', data: { value: 1 } } }, }), ], }); @@ -1394,7 +1394,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], bbb: 'bbb@2-0', ccc: 'ccc@5-0', }, - entries: { + materialisedEntries: { foo1: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, foo2: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, foo3: { timeserial: 'ccc@5-0', data: { value: 'bar' } }, @@ -1410,13 +1410,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], siteTimeserials: { bbb: 'bbb@1-0', }, - count: 1, + initialCount: 1, }), // add objects to the root so they're discoverable in the state tree liveObjectsHelper.mapObject({ objectId: 'root', siteTimeserials: { '000': '000@0-0' }, - entries: { + initialEntries: { map: { timeserial: '000@0-0', data: { objectId: mapId } }, counter: { timeserial: '000@0-0', data: { objectId: counterId } }, }, From 245b6a2f73591c37684c431c240f023281640504 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 22 Nov 2024 17:20:44 +0000 Subject: [PATCH 069/166] Don't use `this` in static methods in StateMessage --- src/plugins/liveobjects/statemessage.ts | 39 ++++++++++++++++++------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index f271ef59e8..cadcccca81 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -151,23 +151,23 @@ export class StateMessage { // TODO: decide how to handle individual errors from decoding values. currently we throw first ever error we get if (message.object?.map?.entries) { - await this._decodeMapEntries(message.object.map.entries, inputContext, decodeDataFn); + await StateMessage._decodeMapEntries(message.object.map.entries, inputContext, decodeDataFn); } if (message.object?.createOp?.map?.entries) { - await this._decodeMapEntries(message.object.createOp.map.entries, inputContext, decodeDataFn); + await StateMessage._decodeMapEntries(message.object.createOp.map.entries, inputContext, decodeDataFn); } if (message.object?.createOp?.mapOp?.data && 'value' in message.object.createOp.mapOp.data) { - await this._decodeStateData(message.object.createOp.mapOp.data, inputContext, decodeDataFn); + await StateMessage._decodeStateData(message.object.createOp.mapOp.data, inputContext, decodeDataFn); } if (message.operation?.map?.entries) { - await this._decodeMapEntries(message.operation.map.entries, inputContext, decodeDataFn); + await StateMessage._decodeMapEntries(message.operation.map.entries, inputContext, decodeDataFn); } if (message.operation?.mapOp?.data && 'value' in message.operation.mapOp.data) { - await this._decodeStateData(message.operation.mapOp.data, inputContext, decodeDataFn); + await StateMessage._decodeStateData(message.operation.mapOp.data, inputContext, decodeDataFn); } } @@ -180,7 +180,7 @@ export class StateMessage { const result = new Array(count); for (let i = 0; i < count; i++) { - result[i] = this.fromValues(values[i] as Record, platform); + result[i] = StateMessage.fromValues(values[i] as Record, platform); } return result; @@ -192,7 +192,7 @@ export class StateMessage { decodeDataFn: typeof decodeData, ): Promise { for (const entry of Object.values(mapEntries)) { - await this._decodeStateData(entry.data, inputContext, decodeDataFn); + await StateMessage._decodeStateData(entry.data, inputContext, decodeDataFn); } } @@ -221,13 +221,21 @@ export class StateMessage { if (stateOperationCopy.mapOp?.data && 'value' in stateOperationCopy.mapOp.data) { // use original "stateOperation" object when encoding values, so we have access to the original buffer values. - stateOperationCopy.mapOp.data = this._encodeStateData(platform, stateOperation.mapOp?.data!, withBase64Encoding); + stateOperationCopy.mapOp.data = StateMessage._encodeStateData( + platform, + stateOperation.mapOp?.data!, + withBase64Encoding, + ); } if (stateOperationCopy.map?.entries) { Object.entries(stateOperationCopy.map.entries).forEach(([key, entry]) => { // use original "stateOperation" object when encoding values, so we have access to original buffer values. - entry.data = this._encodeStateData(platform, stateOperation?.map?.entries?.[key].data!, withBase64Encoding); + entry.data = StateMessage._encodeStateData( + platform, + stateOperation?.map?.entries?.[key].data!, + withBase64Encoding, + ); }); } @@ -256,14 +264,23 @@ export class StateMessage { if (stateObjectCopy.createOp) { // use original "stateObject" object when encoding values, so we have access to original buffer values. - stateObjectCopy.createOp = this._encodeStateOperation(platform, stateObject.createOp!, withBase64Encoding); + stateObjectCopy.createOp = StateMessage._encodeStateOperation( + platform, + stateObject.createOp!, + withBase64Encoding, + ); } return stateObjectCopy; } private static _encodeStateData(platform: typeof Platform, data: StateData, withBase64Encoding: boolean): StateData { - const { value, encoding } = this._encodeStateValue(platform, data?.value, data?.encoding, withBase64Encoding); + const { value, encoding } = StateMessage._encodeStateValue( + platform, + data?.value, + data?.encoding, + withBase64Encoding, + ); return { ...data, value, From 0b4b1e4d80fd18da88056c976f21fcdbcc5e8743 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 19 Nov 2024 03:18:28 +0000 Subject: [PATCH 070/166] Add LiveObjects access and subscription public API to the `ably.d.ts` This is needed to enable LiveObjects plugin package tests and TypeScript checks. --- ably.d.ts | 126 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 125 insertions(+), 1 deletion(-) diff --git a/ably.d.ts b/ably.d.ts index 2f3dd499af..2fefb6d8f2 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -1559,6 +1559,13 @@ export type DeregisterCallback = (device: DeviceDetails, callback: StandardCallb */ export type ErrorCallback = (error: ErrorInfo | null) => void; +/** + * A callback used in {@link LiveObject} to listen for updates to the Live Object. + * + * @param update - The update object describing the changes made to the Live Object. + */ +export type LiveObjectUpdateCallback = (update: T) => void; + // Internal Interfaces // To allow a uniform (callback) interface between on and once even in the @@ -2028,7 +2035,124 @@ export declare interface PushChannel { /** * Enables the LiveObjects state to be subscribed to for a channel. */ -export declare interface LiveObjects {} +export declare interface LiveObjects { + /** + * Retrieves the root {@link LiveMap} object for state on a channel. + * + * @returns A promise which, upon success, will be fulfilled with a {@link LiveMap} object. Upon failure, the promise will be rejected with an {@link ErrorInfo} object which explains the error. + */ + getRoot(): Promise; +} + +/** + * The `LiveMap` class represents a synchronized key/value storage, similar to a JavaScript Map, where all changes are synchronized across clients in realtime. + * Conflict-free resolution for updates follows Last Write Wins (LWW) semantics, meaning that if two clients update the same key in the map, the last change wins. + * + * Keys must be strings. Values can be another Live Object, or a primitive type, such as a string, number, boolean, or binary data (see {@link StateValue}). + */ +export declare interface LiveMap extends LiveObject { + /** + * Returns the value associated with a given key. Returns `undefined` if the key doesn't exist in a map. + * + * @param key - The key to retrieve the value for. + * @returns A {@link LiveObject}, a primitive type (string, number, boolean, or binary data) or `undefined` if the key doesn't exist in a map. + */ + get(key: string): LiveObject | StateValue | undefined; + + /** + * Returns the number of key/value pairs in the map. + */ + size(): number; +} + +/** + * Represents an update to a {@link LiveMap} object, describing the keys that were updated or removed. + */ +export declare interface LiveMapUpdate extends LiveObjectUpdate { + /** + * An object containing keys from a `LiveMap` that have changed, along with their change status: + * - `updated` - the value of a key in the map was updated. + * - `removed` - the key was removed from the map. + */ + update: { [keyName: string]: 'updated' | 'removed' }; +} + +/** + * Represents a primitive value that can be stored in a {@link LiveMap}. + * + * For binary data, the resulting type depends on the platform (`Buffer` in Node.js, `ArrayBuffer` elsewhere). + */ +export type StateValue = string | number | boolean | Buffer | ArrayBuffer; + +/** + * The `LiveCounter` class represents a synchronized counter that can be incremented or decremented and is synchronized across clients in realtime. + */ +export declare interface LiveCounter extends LiveObject { + /** + * Returns the current value of the counter. + */ + value(): number; +} + +/** + * Represents an update to a {@link LiveCounter} object. + */ +export declare interface LiveCounterUpdate extends LiveObjectUpdate { + /** + * Holds the numerical change to the counter value. + */ + update: { + /** + * The value by which the counter was incremented or decremented. + */ + inc: number; + }; +} + +/** + * Describes the common interface for all conflict-free data structures supported by the `LiveObjects`. + */ +export declare interface LiveObject { + /** + * Registers a listener that is called each time this Live Object is updated. + * + * @param listener - An event listener function that is called with an update object whenever this Live Object is updated. + * @returns A {@link SubscribeResponse} object that allows the provided listener to be deregistered from future updates. + */ + subscribe(listener: LiveObjectUpdateCallback): SubscribeResponse; + + /** + * Deregisters the given listener from updates for this Live Object. + * + * @param listener - An event listener function. + */ + unsubscribe(listener: LiveObjectUpdateCallback): void; + + /** + * Deregisters all listeners from updates for this Live Object. + */ + unsubscribeAll(): void; +} + +/** + * Represents a generic update object describing the changes that occurred on a Live Object. + */ +export declare interface LiveObjectUpdate { + /** + * Holds an update object which describe changes applied to the object. + */ + update: any; +} + +/** + * Object returned from a `subscribe` call, allowing the listener provided in that call to be deregistered. + */ +export declare interface SubscribeResponse { + /** + * Deregisters the listener passed to the `subscribe` call. + */ + unsubscribe(): void; +} /** * Enables messages to be published and historic messages to be retrieved for a channel. From 687633985754fe7dfdafb897a756a78f56adfdd0 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 25 Oct 2024 11:08:35 +0100 Subject: [PATCH 071/166] Use common `_decodeAndPrepareMessages` for processing of `STATE` and `STATE_SYNC` messages This implements the proposal from one of the earlier PRs [1] [1] https://github.com/ably/ably-js/pull/1897#discussion_r1814781743 --- src/common/lib/client/realtimechannel.ts | 70 +++++------------------- 1 file changed, 14 insertions(+), 56 deletions(-) diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index 1b75fc06e5..427b6398e6 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -30,7 +30,7 @@ import { ChannelOptions } from '../../types/channel'; import { normaliseChannelOptions } from '../util/defaults'; import { PaginatedResult } from './paginatedresource'; import type { PushChannel } from 'plugins/push'; -import type { LiveObjects } from 'plugins/liveobjects'; +import type { LiveObjects, StateMessage } from 'plugins/liveobjects'; interface RealtimeHistoryParams { start?: number; @@ -604,69 +604,27 @@ class RealtimeChannel extends EventEmitter { break; } - case actions.STATE: { - if (!this._liveObjects) { - return; - } - - const { id, connectionId, timestamp } = message; - const options = this.channelOptions; - - const stateMessages = message.state ?? []; - for (let i = 0; i < stateMessages.length; i++) { - try { - const stateMessage = stateMessages[i]; - - await this.client._LiveObjectsPlugin?.StateMessage.decode(stateMessage, options, decodeData); - - if (!stateMessage.connectionId) stateMessage.connectionId = connectionId; - if (!stateMessage.timestamp) stateMessage.timestamp = timestamp; - if (!stateMessage.id) stateMessage.id = id + ':' + i; - } catch (e) { - Logger.logAction( - this.logger, - Logger.LOG_ERROR, - 'RealtimeChannel.processMessage()', - (e as Error).toString(), - ); - } - } - - this._liveObjects.handleStateMessages(stateMessages); - - break; - } - + // STATE and STATE_SYNC message processing share most of the logic, so group them together + case actions.STATE: case actions.STATE_SYNC: { if (!this._liveObjects) { return; } - const { id, connectionId, timestamp } = message; + const stateMessages = message.state ?? []; const options = this.channelOptions; + await this._decodeAndPrepareMessages(message, stateMessages, (msg) => + this.client._LiveObjectsPlugin + ? this.client._LiveObjectsPlugin.StateMessage.decode(msg, options, decodeData) + : Utils.throwMissingPluginError('LiveObjects'), + ); - const stateMessages = message.state ?? []; - for (let i = 0; i < stateMessages.length; i++) { - try { - const stateMessage = stateMessages[i]; - - await this.client._LiveObjectsPlugin?.StateMessage.decode(stateMessage, options, decodeData); - - if (!stateMessage.connectionId) stateMessage.connectionId = connectionId; - if (!stateMessage.timestamp) stateMessage.timestamp = timestamp; - if (!stateMessage.id) stateMessage.id = id + ':' + i; - } catch (e) { - Logger.logAction( - this.logger, - Logger.LOG_ERROR, - 'RealtimeChannel.processMessage()', - (e as Error).toString(), - ); - } + if (message.action === actions.STATE) { + this._liveObjects.handleStateMessages(stateMessages); + } else { + this._liveObjects.handleStateSyncMessages(stateMessages, message.channelSerial); } - this._liveObjects.handleStateSyncMessages(stateMessages, message.channelSerial); - break; } @@ -774,7 +732,7 @@ class RealtimeChannel extends EventEmitter { * @returns `unrecoverableError` flag. If `true` indicates that unrecoverable error was encountered during message decoding * and any further message processing should be stopped. Always equals to `false` if `decodeErrorRecoveryHandler` was not provided */ - private async _decodeAndPrepareMessages( + private async _decodeAndPrepareMessages( protocolMessage: ProtocolMessage, messages: T[], decodeFn: (msg: T) => Promise, From 6cbe7ed85def6f1e7d482248c4f468131be13b87 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 29 Nov 2024 06:19:51 +0000 Subject: [PATCH 072/166] Apply docstring changes suggested in the code review Co-authored-by: Mike Christensen --- ably.d.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ably.d.ts b/ably.d.ts index 2fefb6d8f2..225fc806cc 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -2045,8 +2045,8 @@ export declare interface LiveObjects { } /** - * The `LiveMap` class represents a synchronized key/value storage, similar to a JavaScript Map, where all changes are synchronized across clients in realtime. - * Conflict-free resolution for updates follows Last Write Wins (LWW) semantics, meaning that if two clients update the same key in the map, the last change wins. + * The `LiveMap` class represents a key/value map data structure, similar to a JavaScript Map, where all changes are synchronized across clients in realtime. + * Conflict-free resolution for updates follows Last Write Wins (LWW) semantics, meaning that if two clients update the same key in the map, the update with the most recent timestamp wins. * * Keys must be strings. Values can be another Live Object, or a primitive type, such as a string, number, boolean, or binary data (see {@link StateValue}). */ @@ -2085,7 +2085,7 @@ export declare interface LiveMapUpdate extends LiveObjectUpdate { export type StateValue = string | number | boolean | Buffer | ArrayBuffer; /** - * The `LiveCounter` class represents a synchronized counter that can be incremented or decremented and is synchronized across clients in realtime. + * The `LiveCounter` class represents a counter that can be incremented or decremented and is synchronized across clients in realtime. */ export declare interface LiveCounter extends LiveObject { /** From c6f8d2eb9d009db77a5a14b409914e24321b5202 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 19 Nov 2024 03:16:01 +0000 Subject: [PATCH 073/166] Add LiveObjects plugin test to the `package` tests Test LiveObjects plugin can be imported and provided to the Ably client, and that TypeScript types for LiveObjects work as expected. --- test/package/browser/template/README.md | 4 +- test/package/browser/template/package.json | 2 +- .../server/resources/index-liveobjects.html | 11 +++++ .../package/browser/template/server/server.ts | 2 +- .../browser/template/src/index-liveobjects.ts | 43 +++++++++++++++++++ test/package/browser/template/src/sandbox.ts | 8 +++- .../browser/template/test/lib/package.test.ts | 1 + 7 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 test/package/browser/template/server/resources/index-liveobjects.html create mode 100644 test/package/browser/template/src/index-liveobjects.ts diff --git a/test/package/browser/template/README.md b/test/package/browser/template/README.md index 8bf44061be..38f2e248d2 100644 --- a/test/package/browser/template/README.md +++ b/test/package/browser/template/README.md @@ -8,6 +8,7 @@ This directory is intended to be used for testing the following aspects of the a It contains three files, each of which import ably-js in different manners, and provide a way to briefly exercise its functionality: - `src/index-default.ts` imports the default ably-js package (`import { Realtime } from 'ably'`). +- `src/index-liveobjects.ts` imports the LiveObjects ably-js plugin (`import LiveObjects from 'ably/liveobjects'`). - `src/index-modular.ts` imports the tree-shakable ably-js package (`import { BaseRealtime, WebSocketTransport, FetchRequest } from 'ably/modular'`). - `src/ReactApp.tsx` imports React hooks from the ably-js package (`import { useChannel } from 'ably/react'`). @@ -25,6 +26,7 @@ This directory exposes three package scripts that are to be used for testing: - `build`: Uses esbuild to create: 1. a bundle containing `src/index-default.ts` and ably-js; - 2. a bundle containing `src/index-modular.ts` and ably-js. + 2. a bundle containing `src/index-liveobjects.ts` and ably-js. + 3. a bundle containing `src/index-modular.ts` and ably-js. - `test`: Using the bundles created by `build` and playwright components setup, tests that the code that exercises ably-js’s functionality is working correctly in a browser. - `typecheck`: Type-checks the code that imports ably-js. diff --git a/test/package/browser/template/package.json b/test/package/browser/template/package.json index a05aa04977..f2c023b6e6 100644 --- a/test/package/browser/template/package.json +++ b/test/package/browser/template/package.json @@ -4,7 +4,7 @@ "description": "", "main": "index.js", "scripts": { - "build": "esbuild --bundle src/index-default.ts --outdir=dist && esbuild --bundle src/index-modular.ts --outdir=dist", + "build": "esbuild --bundle src/index-default.ts --outdir=dist && esbuild --bundle src/index-liveobjects.ts --outdir=dist && esbuild --bundle src/index-modular.ts --outdir=dist", "typecheck": "tsc --project src -noEmit", "test-support:server": "ts-node server/server.ts", "test": "npm run test:lib && npm run test:hooks", diff --git a/test/package/browser/template/server/resources/index-liveobjects.html b/test/package/browser/template/server/resources/index-liveobjects.html new file mode 100644 index 0000000000..b7f284af5a --- /dev/null +++ b/test/package/browser/template/server/resources/index-liveobjects.html @@ -0,0 +1,11 @@ + + + + + Ably NPM package test (LiveObjects plugin export) + + + + + + diff --git a/test/package/browser/template/server/server.ts b/test/package/browser/template/server/server.ts index 2409fc30c5..faa1399f70 100644 --- a/test/package/browser/template/server/server.ts +++ b/test/package/browser/template/server/server.ts @@ -5,7 +5,7 @@ async function startWebServer(listenPort: number) { const server = express(); server.get('/', (req, res) => res.send('OK')); server.use(express.static(path.join(__dirname, '/resources'))); - for (const filename of ['index-default.js', 'index-modular.js']) { + for (const filename of ['index-default.js', 'index-liveobjects.js', 'index-modular.js']) { server.use(`/${filename}`, express.static(path.join(__dirname, '..', 'dist', filename))); } diff --git a/test/package/browser/template/src/index-liveobjects.ts b/test/package/browser/template/src/index-liveobjects.ts new file mode 100644 index 0000000000..9334aadf42 --- /dev/null +++ b/test/package/browser/template/src/index-liveobjects.ts @@ -0,0 +1,43 @@ +import * as Ably from 'ably'; +import LiveObjects from 'ably/liveobjects'; +import { createSandboxAblyAPIKey } from './sandbox'; + +globalThis.testAblyPackage = async function () { + const key = await createSandboxAblyAPIKey({ featureFlags: ['enableChannelState'] }); + + const realtime = new Ably.Realtime({ key, environment: 'sandbox', plugins: { LiveObjects } }); + + const channel = realtime.channels.get('channel', { modes: ['STATE_SUBSCRIBE', 'STATE_PUBLISH'] }); + // check liveObjects can be accessed + const liveObjects = channel.liveObjects; + await channel.attach(); + // root should be a LiveMap object + const root: Ably.LiveMap = await liveObjects.getRoot(); + + // check root is recognized as LiveMap TypeScript type + root.get('someKey'); + root.size(); + + // check LiveMap subscription callback has correct TypeScript types + const { unsubscribe } = root.subscribe(({ update }) => { + switch (update.someKey) { + case 'removed': + case 'updated': + break; + default: + // check all possible types are exhausted + const shouldExhaustAllTypes: never = update.someKey; + } + }); + unsubscribe(); + + // check LiveCounter types also behave as expected + const counter = root.get('randomKey') as Ably.LiveCounter | undefined; + // use nullish coalescing as we didn't actually create a counter object on the root, + // so the next calls would fail. we only need to check that TypeScript types work + const value: number = counter?.value(); + const counterSubscribeResponse = counter?.subscribe(({ update }) => { + const shouldBeANumber: number = update.inc; + }); + counterSubscribeResponse?.unsubscribe(); +}; diff --git a/test/package/browser/template/src/sandbox.ts b/test/package/browser/template/src/sandbox.ts index 100ab22f00..54c5f2e64a 100644 --- a/test/package/browser/template/src/sandbox.ts +++ b/test/package/browser/template/src/sandbox.ts @@ -1,10 +1,14 @@ import testAppSetup from '../../../../common/ably-common/test-resources/test-app-setup.json'; -export async function createSandboxAblyAPIKey() { +export async function createSandboxAblyAPIKey(withOptions?: object) { + const postData = { + ...testAppSetup.post_apps, + ...(withOptions ?? {}), + }; const response = await fetch('https://sandbox-rest.ably.io/apps', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(testAppSetup.post_apps), + body: JSON.stringify(postData), }); if (!response.ok) { diff --git a/test/package/browser/template/test/lib/package.test.ts b/test/package/browser/template/test/lib/package.test.ts index fc73006f3d..8554dd762b 100644 --- a/test/package/browser/template/test/lib/package.test.ts +++ b/test/package/browser/template/test/lib/package.test.ts @@ -3,6 +3,7 @@ import { test, expect } from '@playwright/test'; test.describe('NPM package', () => { for (const scenario of [ { name: 'default export', path: '/index-default.html' }, + { name: 'LiveObjects plugin export', path: '/index-liveobjects.html' }, { name: 'modular export', path: '/index-modular.html' }, ]) { test.describe(scenario.name, () => { From d59e16710f51421ccf2dba7f53956eccb1d1283c Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 28 Nov 2024 04:54:08 +0000 Subject: [PATCH 074/166] Fix incorrect `moduleResolution` in `tsconfig.json` for package tests `moduleResolution` must be set to `node` as `resolveJsonModule: true` cannot be used otherwise. --- test/package/browser/template/src/tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/package/browser/template/src/tsconfig.json b/test/package/browser/template/src/tsconfig.json index e280baa6de..b206a63995 100644 --- a/test/package/browser/template/src/tsconfig.json +++ b/test/package/browser/template/src/tsconfig.json @@ -4,7 +4,7 @@ "resolveJsonModule": true, "esModuleInterop": true, "module": "esnext", - "moduleResolution": "bundler", + "moduleResolution": "node", "jsx": "react-jsx" } } From f4113d8e554dca564b33700175b572540e689508 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 19 Nov 2024 03:43:17 +0000 Subject: [PATCH 075/166] Add support for user-provided typings for the LiveObjects data structure Resolves DTP-963 --- src/plugins/liveobjects/livemap.ts | 22 ++++++++++++---------- src/plugins/liveobjects/liveobjects.ts | 10 ++++++++-- src/plugins/liveobjects/typings.ts | 22 ++++++++++++++++++++++ 3 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 src/plugins/liveobjects/typings.ts diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index b29654c02f..d20d1c47fb 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -13,6 +13,7 @@ import { StateValue, } from './statemessage'; import { DefaultTimeserial, Timeserial } from './timeserial'; +import { LiveMapType } from './typings'; export interface ObjectIdStateData { /** A reference to another state object, used to support composable state objects. */ @@ -45,7 +46,7 @@ export interface LiveMapUpdate extends LiveObjectUpdate { update: { [keyName: string]: 'updated' | 'removed' }; } -export class LiveMap extends LiveObject { +export class LiveMap extends LiveObject { constructor( liveObjects: LiveObjects, private _semantics: MapSemantics, @@ -59,8 +60,8 @@ export class LiveMap extends LiveObject { * * @internal */ - static zeroValue(liveobjects: LiveObjects, objectId: string): LiveMap { - return new LiveMap(liveobjects, MapSemantics.LWW, objectId); + static zeroValue(liveobjects: LiveObjects, objectId: string): LiveMap { + return new LiveMap(liveobjects, MapSemantics.LWW, objectId); } /** @@ -69,8 +70,8 @@ export class LiveMap extends LiveObject { * * @internal */ - static fromStateObject(liveobjects: LiveObjects, stateObject: StateObject): LiveMap { - const obj = new LiveMap(liveobjects, stateObject.map?.semantics!, stateObject.objectId); + static fromStateObject(liveobjects: LiveObjects, stateObject: StateObject): LiveMap { + const obj = new LiveMap(liveobjects, stateObject.map?.semantics!, stateObject.objectId); obj.overrideWithStateObject(stateObject); return obj; } @@ -82,24 +83,25 @@ export class LiveMap extends LiveObject { * then you will get a reference to that Live Object if it exists in the local pool, or undefined otherwise. * If the value is not an objectId, then you will get that value. */ - get(key: string): LiveObject | StateValue | undefined { + // force the key to be of type string as we only allow strings as key in a map + get(key: TKey): T[TKey] { const element = this._dataRef.data.get(key); if (element === undefined) { - return undefined; + return undefined as T[TKey]; } if (element.tombstone === true) { - return undefined; + return undefined as T[TKey]; } // data exists for non-tombstoned elements const data = element.data!; if ('value' in data) { - return data.value; + return data.value as T[TKey]; } else { - return this._liveObjects.getPool().get(data.objectId); + return this._liveObjects.getPool().get(data.objectId) as T[TKey]; } } diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 5ba586fe42..741544d55a 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -8,6 +8,7 @@ import { LiveObject, LiveObjectUpdate } from './liveobject'; import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage } from './statemessage'; import { SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; +import { DefaultRoot, LiveMapType } from './typings'; enum LiveObjectsEvents { SyncCompleted = 'SyncCompleted', @@ -36,13 +37,18 @@ export class LiveObjects { this._bufferedStateOperations = []; } - async getRoot(): Promise { + /** + * When called without a type variable, we return a default root type which is based on globally defined LiveObjects interface. + * A user can provide an explicit type for the getRoot method to explicitly set the LiveObjects type structure on this particular channel. + * This is useful when working with LiveObjects on multiple channels with different underlying data. + */ + async getRoot(): Promise> { // SYNC is currently in progress, wait for SYNC sequence to finish if (this._syncInProgress) { await this._eventEmitter.once(LiveObjectsEvents.SyncCompleted); } - return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; + return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; } /** diff --git a/src/plugins/liveobjects/typings.ts b/src/plugins/liveobjects/typings.ts new file mode 100644 index 0000000000..879b6a29c0 --- /dev/null +++ b/src/plugins/liveobjects/typings.ts @@ -0,0 +1,22 @@ +import { LiveCounter } from './livecounter'; +import { LiveMap } from './livemap'; +import { StateValue } from './statemessage'; + +declare global { + // define a global interface which can be used by users to define their own types for LiveObjects. + export interface LiveObjectsTypes { + [key: string]: unknown; + } +} + +// LiveMap type representation of how it looks to the end-user. A mapping of string keys to the scalar values (StateValue) or other Live Objects. +export type LiveMapType = { [key: string]: StateValue | LiveMap | LiveCounter | undefined }; + +export type DefaultRoot = + // we need a way to understand when no types were provided by the user. + // we expect a "root" property to be set on LiveObjectsTypes interface, e.g. it won't be "unknown" anymore + unknown extends LiveObjectsTypes['root'] + ? LiveMapType // no types provided by the user, use the default map type for the root + : LiveObjectsTypes['root'] extends LiveMapType + ? LiveObjectsTypes['root'] // "root" was provided by the user, and it is of an expected type, we can use this interface for the root object in LiveObjects. + : `Provided type definition for the "root" object in LiveObjectsTypes is not of an expected LiveMapType type`; From 73c8abcbda0261392f754e6fb88821a52e0d7a56 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 19 Nov 2024 04:07:41 +0000 Subject: [PATCH 076/166] Move types required for user-provided LiveObjects typings to ably.d.ts Add globally defined `LiveObjectsTypes` to `intentionallyNotExported` [1] list for `typedoc`, as typedocs does not include documentation for the globally defined interfaces and complains that "__global.LiveObjectsTypes, defined in ./ably.d.ts, is referenced by ably.DefaultRoot but not included in the documentation." [1] https://typedoc.org/documents/Options.Validation.html#intentionallynotexported --- ably.d.ts | 57 ++++++++++++++++++++++++-- src/plugins/liveobjects/livemap.ts | 8 ++-- src/plugins/liveobjects/liveobjects.ts | 3 +- src/plugins/liveobjects/typings.ts | 22 ---------- typedoc.json | 3 +- 5 files changed, 61 insertions(+), 32 deletions(-) delete mode 100644 src/plugins/liveobjects/typings.ts diff --git a/ably.d.ts b/ably.d.ts index 225fc806cc..5f375b1eca 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -2039,25 +2039,76 @@ export declare interface LiveObjects { /** * Retrieves the root {@link LiveMap} object for state on a channel. * + * A type parameter can be provided to describe the structure of the LiveObjects state on the channel. By default, it uses types from the globally defined `LiveObjectsTypes` interface. + * + * You can specify custom types for LiveObjects by defining a global `LiveObjectsTypes` interface with a `root` property that conforms to {@link LiveMapType}. + * + * Example: + * + * ```typescript + * import { LiveCounter } from 'ably'; + * + * type MyRoot = { + * myTypedKey: LiveCounter; + * }; + * + * declare global { + * export interface LiveObjectsTypes { + * root: MyRoot; + * } + * } + * ``` + * * @returns A promise which, upon success, will be fulfilled with a {@link LiveMap} object. Upon failure, the promise will be rejected with an {@link ErrorInfo} object which explains the error. */ - getRoot(): Promise; + getRoot(): Promise>; } +declare global { + /** + * A globally defined interface that allows users to define custom types for LiveObjects. + */ + export interface LiveObjectsTypes { + [key: string]: unknown; + } +} + +/** + * Represents the type of data stored in a {@link LiveMap}. + * It maps string keys to scalar values ({@link StateValue}), or other LiveObjects. + */ +export type LiveMapType = { [key: string]: StateValue | LiveMap | LiveCounter | undefined }; + +/** + * The default type for the `root` object in the LiveObjects, based on the globally defined {@link LiveObjectsTypes} interface. + * + * - If no custom types are provided in `LiveObjectsTypes`, defaults to an untyped root map representation using the {@link LiveMapType} interface. + * - If a `root` type exists in `LiveObjectsTypes` and conforms to the {@link LiveMapType} interface, it is used as the type for the `root` object. + * - If the provided `root` type does not match {@link LiveMapType}, a type error message is returned. + */ +export type DefaultRoot = + // we need a way to know when no types were provided by the user. + // we expect a "root" property to be set on LiveObjectsTypes interface, e.g. it won't be "unknown" anymore + unknown extends LiveObjectsTypes['root'] + ? LiveMapType // no custom types provided; use the default untyped map representation for the root + : LiveObjectsTypes['root'] extends LiveMapType + ? LiveObjectsTypes['root'] // "root" property exists, and it is of an expected type, we can use this interface for the root object in LiveObjects. + : `Provided type definition for the "root" object in LiveObjectsTypes is not of an expected LiveMapType`; + /** * The `LiveMap` class represents a key/value map data structure, similar to a JavaScript Map, where all changes are synchronized across clients in realtime. * Conflict-free resolution for updates follows Last Write Wins (LWW) semantics, meaning that if two clients update the same key in the map, the update with the most recent timestamp wins. * * Keys must be strings. Values can be another Live Object, or a primitive type, such as a string, number, boolean, or binary data (see {@link StateValue}). */ -export declare interface LiveMap extends LiveObject { +export declare interface LiveMap extends LiveObject { /** * Returns the value associated with a given key. Returns `undefined` if the key doesn't exist in a map. * * @param key - The key to retrieve the value for. * @returns A {@link LiveObject}, a primitive type (string, number, boolean, or binary data) or `undefined` if the key doesn't exist in a map. */ - get(key: string): LiveObject | StateValue | undefined; + get(key: TKey): T[TKey]; /** * Returns the number of key/value pairs in the map. diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index d20d1c47fb..07577d4b2c 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -1,5 +1,6 @@ import deepEqual from 'deep-equal'; +import type * as API from '../../../ably'; import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; import { @@ -13,7 +14,6 @@ import { StateValue, } from './statemessage'; import { DefaultTimeserial, Timeserial } from './timeserial'; -import { LiveMapType } from './typings'; export interface ObjectIdStateData { /** A reference to another state object, used to support composable state objects. */ @@ -46,7 +46,7 @@ export interface LiveMapUpdate extends LiveObjectUpdate { update: { [keyName: string]: 'updated' | 'removed' }; } -export class LiveMap extends LiveObject { +export class LiveMap extends LiveObject { constructor( liveObjects: LiveObjects, private _semantics: MapSemantics, @@ -60,7 +60,7 @@ export class LiveMap extends LiveObject(liveobjects: LiveObjects, objectId: string): LiveMap { + static zeroValue(liveobjects: LiveObjects, objectId: string): LiveMap { return new LiveMap(liveobjects, MapSemantics.LWW, objectId); } @@ -70,7 +70,7 @@ export class LiveMap extends LiveObject(liveobjects: LiveObjects, stateObject: StateObject): LiveMap { + static fromStateObject(liveobjects: LiveObjects, stateObject: StateObject): LiveMap { const obj = new LiveMap(liveobjects, stateObject.map?.semantics!, stateObject.objectId); obj.overrideWithStateObject(stateObject); return obj; diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 741544d55a..75b743d5b0 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -8,7 +8,6 @@ import { LiveObject, LiveObjectUpdate } from './liveobject'; import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage } from './statemessage'; import { SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; -import { DefaultRoot, LiveMapType } from './typings'; enum LiveObjectsEvents { SyncCompleted = 'SyncCompleted', @@ -42,7 +41,7 @@ export class LiveObjects { * A user can provide an explicit type for the getRoot method to explicitly set the LiveObjects type structure on this particular channel. * This is useful when working with LiveObjects on multiple channels with different underlying data. */ - async getRoot(): Promise> { + async getRoot(): Promise> { // SYNC is currently in progress, wait for SYNC sequence to finish if (this._syncInProgress) { await this._eventEmitter.once(LiveObjectsEvents.SyncCompleted); diff --git a/src/plugins/liveobjects/typings.ts b/src/plugins/liveobjects/typings.ts deleted file mode 100644 index 879b6a29c0..0000000000 --- a/src/plugins/liveobjects/typings.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { LiveCounter } from './livecounter'; -import { LiveMap } from './livemap'; -import { StateValue } from './statemessage'; - -declare global { - // define a global interface which can be used by users to define their own types for LiveObjects. - export interface LiveObjectsTypes { - [key: string]: unknown; - } -} - -// LiveMap type representation of how it looks to the end-user. A mapping of string keys to the scalar values (StateValue) or other Live Objects. -export type LiveMapType = { [key: string]: StateValue | LiveMap | LiveCounter | undefined }; - -export type DefaultRoot = - // we need a way to understand when no types were provided by the user. - // we expect a "root" property to be set on LiveObjectsTypes interface, e.g. it won't be "unknown" anymore - unknown extends LiveObjectsTypes['root'] - ? LiveMapType // no types provided by the user, use the default map type for the root - : LiveObjectsTypes['root'] extends LiveMapType - ? LiveObjectsTypes['root'] // "root" was provided by the user, and it is of an expected type, we can use this interface for the root object in LiveObjects. - : `Provided type definition for the "root" object in LiveObjectsTypes is not of an expected LiveMapType type`; diff --git a/typedoc.json b/typedoc.json index 24faf3bad9..7c9c174766 100644 --- a/typedoc.json +++ b/typedoc.json @@ -20,5 +20,6 @@ "TypeAlias", "Variable", "Namespace" - ] + ], + "intentionallyNotExported": ["__global.LiveObjectsTypes"] } From 3938b639ecd726e90eee2dd49c6507e43a07a8bd Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 19 Nov 2024 04:31:36 +0000 Subject: [PATCH 077/166] Add tests for user-provided LiveObjects types to the LiveObjects package test --- .../browser/template/src/ably.config.d.ts | 21 ++++++++++ .../browser/template/src/index-liveobjects.ts | 42 ++++++++++++++----- .../browser/template/src/tsconfig.json | 1 + 3 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 test/package/browser/template/src/ably.config.d.ts diff --git a/test/package/browser/template/src/ably.config.d.ts b/test/package/browser/template/src/ably.config.d.ts new file mode 100644 index 0000000000..e5bca7718f --- /dev/null +++ b/test/package/browser/template/src/ably.config.d.ts @@ -0,0 +1,21 @@ +import { LiveCounter, LiveMap } from 'ably'; + +type CustomRoot = { + numberKey: number; + stringKey: string; + booleanKey: boolean; + couldBeUndefined?: string; + mapKey?: LiveMap<{ + foo: 'bar'; + nestedMap?: LiveMap<{ + baz: 'qux'; + }>; + }>; + counterKey?: LiveCounter; +}; + +declare global { + export interface LiveObjectsTypes { + root: CustomRoot; + } +} diff --git a/test/package/browser/template/src/index-liveobjects.ts b/test/package/browser/template/src/index-liveobjects.ts index 9334aadf42..1cd27b0217 100644 --- a/test/package/browser/template/src/index-liveobjects.ts +++ b/test/package/browser/template/src/index-liveobjects.ts @@ -1,7 +1,12 @@ import * as Ably from 'ably'; import LiveObjects from 'ably/liveobjects'; +import { CustomRoot } from './ably.config'; import { createSandboxAblyAPIKey } from './sandbox'; +type ExplicitRootType = { + someOtherKey: string; +}; + globalThis.testAblyPackage = async function () { const key = await createSandboxAblyAPIKey({ featureFlags: ['enableChannelState'] }); @@ -11,12 +16,27 @@ globalThis.testAblyPackage = async function () { // check liveObjects can be accessed const liveObjects = channel.liveObjects; await channel.attach(); - // root should be a LiveMap object - const root: Ably.LiveMap = await liveObjects.getRoot(); + // expect root to be a LiveMap instance with LiveObjects types defined via the global LiveObjectsTypes interface + // also checks that we can refer to the LiveObjects types exported from 'ably' by referencing a LiveMap interface + const root: Ably.LiveMap = await liveObjects.getRoot(); + + // check root has expected LiveMap TypeScript type methods + const size: number = root.size(); - // check root is recognized as LiveMap TypeScript type - root.get('someKey'); - root.size(); + // check custom user provided typings via LiveObjectsTypes are working: + // keys on a root: + const aNumber: number = root.get('numberKey'); + const aString: string = root.get('stringKey'); + const aBoolean: boolean = root.get('booleanKey'); + const couldBeUndefined: string | undefined = root.get('couldBeUndefined'); + // live objects on a root: + const counter: Ably.LiveCounter | undefined = root.get('counterKey'); + const map: LiveObjectsTypes['root']['mapKey'] = root.get('mapKey'); + // check string literal types works + // need to use nullish coalescing as we didn't actually create any data on the root, + // so the next calls would fail. we only need to check that TypeScript types work + const foo: 'bar' = map?.get('foo')!; + const baz: 'qux' = map?.get('nestedMap')?.get('baz')!; // check LiveMap subscription callback has correct TypeScript types const { unsubscribe } = root.subscribe(({ update }) => { @@ -31,13 +51,15 @@ globalThis.testAblyPackage = async function () { }); unsubscribe(); - // check LiveCounter types also behave as expected - const counter = root.get('randomKey') as Ably.LiveCounter | undefined; - // use nullish coalescing as we didn't actually create a counter object on the root, - // so the next calls would fail. we only need to check that TypeScript types work - const value: number = counter?.value(); + // check LiveCounter type also behaves as expected + // same deal with nullish coalescing + const value: number = counter?.value()!; const counterSubscribeResponse = counter?.subscribe(({ update }) => { const shouldBeANumber: number = update.inc; }); counterSubscribeResponse?.unsubscribe(); + + // check can provide custom types for the getRoot method, ignoring global LiveObjectsTypes interface + const explicitRoot: Ably.LiveMap = await liveObjects.getRoot(); + const someOtherKey: string = explicitRoot.get('someOtherKey'); }; diff --git a/test/package/browser/template/src/tsconfig.json b/test/package/browser/template/src/tsconfig.json index b206a63995..3230e8697f 100644 --- a/test/package/browser/template/src/tsconfig.json +++ b/test/package/browser/template/src/tsconfig.json @@ -1,6 +1,7 @@ { "include": ["**/*.ts", "**/*.tsx"], "compilerOptions": { + "strictNullChecks": true, "resolveJsonModule": true, "esModuleInterop": true, "module": "esnext", From 998850462a093957654957d4fba3059f00f32149 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 27 Nov 2024 07:32:05 +0000 Subject: [PATCH 078/166] Use lexico timeserials and `siteCode` field in StateMessages Resolves DTP-1078 --- scripts/moduleReport.ts | 1 - src/plugins/liveobjects/livecounter.ts | 13 +- src/plugins/liveobjects/livemap.ts | 73 ++-- src/plugins/liveobjects/liveobject.ts | 28 +- src/plugins/liveobjects/statemessage.ts | 8 +- src/plugins/liveobjects/timeserial.ts | 190 ---------- test/common/modules/live_objects_helper.js | 7 +- test/realtime/live_objects.test.js | 401 +++++++++++---------- 8 files changed, 291 insertions(+), 430 deletions(-) delete mode 100644 src/plugins/liveobjects/timeserial.ts diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index 0eae4017a2..8d1921789f 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -332,7 +332,6 @@ async function checkLiveObjectsPluginFiles() { 'src/plugins/liveobjects/objectid.ts', 'src/plugins/liveobjects/statemessage.ts', 'src/plugins/liveobjects/syncliveobjectsdatapool.ts', - 'src/plugins/liveobjects/timeserial.ts', ]); return checkBundleFiles(pluginBundleInfo, allowedFiles, 100); diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index bfa3a99cce..dc144c3259 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -1,7 +1,6 @@ import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; import { StateCounterOp, StateMessage, StateObject, StateOperation, StateOperationAction } from './statemessage'; -import { DefaultTimeserial } from './timeserial'; export interface LiveCounterData extends LiveObjectData { data: number; @@ -49,19 +48,20 @@ export class LiveCounter extends LiveObject ); } - const opOriginTimeserial = DefaultTimeserial.calculateTimeserial(this._client, msg.serial); - if (!this._canApplyOperation(opOriginTimeserial)) { + const opOriginTimeserial = msg.serial!; + const opSiteCode = msg.siteCode!; + if (!this._canApplyOperation(opOriginTimeserial, opSiteCode)) { this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MICRO, 'LiveCounter.applyOperation()', - `skipping ${op.action} op: op timeserial ${opOriginTimeserial.toString()} <= site timeserial ${this._siteTimeserials[opOriginTimeserial.siteCode].toString()}; objectId=${this._objectId}`, + `skipping ${op.action} op: op timeserial ${opOriginTimeserial.toString()} <= site timeserial ${this._siteTimeserials[opSiteCode]?.toString()}; objectId=${this._objectId}`, ); return; } // should update stored site timeserial immediately. doesn't matter if we successfully apply the op, // as it's important to mark that the op was processed by the object - this._siteTimeserials[opOriginTimeserial.siteCode] = opOriginTimeserial; + this._siteTimeserials[opSiteCode] = opOriginTimeserial; let update: LiveCounterUpdate | LiveObjectUpdateNoop; switch (op.action) { @@ -125,7 +125,8 @@ export class LiveCounter extends LiveObject // override all relevant data for this object with data from the state object this._createOperationIsMerged = false; this._dataRef = { data: stateObject.counter?.count ?? 0 }; - this._siteTimeserials = this._timeserialMapFromStringMap(stateObject.siteTimeserials); + // should default to empty map if site timeserials do not exist on the state object, so that any future operation can be applied to this object + this._siteTimeserials = stateObject.siteTimeserials ?? {}; if (!this._client.Utils.isNil(stateObject.createOp)) { this._mergeInitialDataFromCreateOperation(stateObject.createOp); } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 07577d4b2c..1f3729c5f0 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -13,7 +13,6 @@ import { StateOperationAction, StateValue, } from './statemessage'; -import { DefaultTimeserial, Timeserial } from './timeserial'; export interface ObjectIdStateData { /** A reference to another state object, used to support composable state objects. */ @@ -34,7 +33,7 @@ export type StateData = ObjectIdStateData | ValueStateData; export interface MapEntry { tombstone: boolean; - timeserial: Timeserial; + timeserial: string | undefined; data: StateData | undefined; } @@ -131,19 +130,20 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject { // for MAP_CREATE op we must use dedicated timeserial field available on an entry, instead of a timeserial on a message - const opOriginTimeserial = entry.timeserial - ? DefaultTimeserial.calculateTimeserial(this._client, entry.timeserial) - : DefaultTimeserial.zeroValueTimeserial(this._client); + const opOriginTimeserial = entry.timeserial; let update: LiveMapUpdate | LiveObjectUpdateNoop; if (entry.tombstone === true) { // entry in MAP_CREATE op is removed, try to apply MAP_REMOVE op @@ -370,20 +369,17 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject entryTimeserial; + } + private _liveMapDataFromMapEntries(entries: Record): LiveMapData { const liveMapData: LiveMapData = { data: new Map(), @@ -470,10 +494,7 @@ export class LiveMap extends LiveObject; + protected _siteTimeserials: Record; protected _createOperationIsMerged: boolean; protected constructor( @@ -106,22 +105,17 @@ export abstract class LiveObject< * An operation should be applied if the origin timeserial is strictly greater than the timeserial in the site timeserials for the same site. * If the site timeserials do not contain a timeserial for the site of the origin timeserial, the operation should be applied. */ - protected _canApplyOperation(opOriginTimeserial: Timeserial): boolean { - const siteTimeserial = this._siteTimeserials[opOriginTimeserial.siteCode]; - return !siteTimeserial || opOriginTimeserial.after(siteTimeserial); - } + protected _canApplyOperation(opOriginTimeserial: string | undefined, opSiteCode: string | undefined): boolean { + if (!opOriginTimeserial) { + throw new this._client.ErrorInfo(`Invalid timeserial: ${opOriginTimeserial}`, 50000, 500); + } + + if (!opSiteCode) { + throw new this._client.ErrorInfo(`Invalid site code: ${opSiteCode}`, 50000, 500); + } - protected _timeserialMapFromStringMap(stringTimeserialsMap: Record): Record { - const objTimeserialsMap = Object.entries(stringTimeserialsMap).reduce( - (acc, v) => { - const [key, timeserialString] = v; - acc[key] = DefaultTimeserial.calculateTimeserial(this._client, timeserialString); - return acc; - }, - {} as Record, - ); - - return objTimeserialsMap; + const siteTimeserial = this._siteTimeserials[opSiteCode]; + return !siteTimeserial || opOriginTimeserial > siteTimeserial; } private _createObjectId(): string { diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index cadcccca81..74b3856300 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -51,8 +51,8 @@ export interface StateMapEntry { /** * The *origin* timeserial of the last operation that was applied to the map entry. * - * It is optional in a MAP_CREATE operation and might be missing, in which case the client should default to using zero-value timeserial, - * which is the "earliest possible" timeserial. This will allow any other operation to update the field based on a timeserial comparison. + * It is optional in a MAP_CREATE operation and might be missing, in which case the client should use a nullish value for it + * and treat it as the "earliest possible" timeserial for comparison purposes. */ timeserial?: string; /** The data that represents the value of the map entry. */ @@ -140,6 +140,8 @@ export class StateMessage { object?: StateObject; /** Timeserial format. Contains the origin timeserial for this state message. */ serial?: string; + /** Site code corresponding to this message's timeserial */ + siteCode?: string; constructor(private _platform: typeof Platform) {} @@ -357,12 +359,14 @@ export class StateMessage { if (this.timestamp) result += '; timestamp=' + this.timestamp; if (this.clientId) result += '; clientId=' + this.clientId; if (this.connectionId) result += '; connectionId=' + this.connectionId; + if (this.channel) result += '; channel=' + this.channel; // TODO: prettify output for operation and object and encode buffers. // see examples for data in Message and PresenceMessage if (this.operation) result += '; operation=' + JSON.stringify(this.operation); if (this.object) result += '; object=' + JSON.stringify(this.object); if (this.extras) result += '; extras=' + JSON.stringify(this.extras); if (this.serial) result += '; serial=' + this.serial; + if (this.siteCode) result += '; siteCode=' + this.siteCode; result += ']'; diff --git a/src/plugins/liveobjects/timeserial.ts b/src/plugins/liveobjects/timeserial.ts deleted file mode 100644 index bc5c535505..0000000000 --- a/src/plugins/liveobjects/timeserial.ts +++ /dev/null @@ -1,190 +0,0 @@ -import type BaseClient from 'common/lib/client/baseclient'; - -/** - * Represents a parsed timeserial. - */ -export interface Timeserial { - /** - * The series ID of the timeserial. - */ - readonly seriesId: string; - - /** - * The site code of the timeserial. - */ - readonly siteCode: string; - - /** - * The timestamp of the timeserial. - */ - readonly timestamp: number; - - /** - * The counter of the timeserial. - */ - readonly counter: number; - - /** - * The index of the timeserial. - */ - readonly index?: number; - - toString(): string; - - before(timeserial: Timeserial | string): boolean; - - after(timeserial: Timeserial | string): boolean; - - equal(timeserial: Timeserial | string): boolean; -} - -/** - * Default implementation of the Timeserial interface. Used internally to parse and compare timeserials. - * - * @internal - */ -export class DefaultTimeserial implements Timeserial { - public readonly seriesId: string; - public readonly siteCode: string; - public readonly timestamp: number; - public readonly counter: number; - public readonly index?: number; - - private constructor( - private _client: BaseClient, - seriesId: string, - timestamp: number, - counter: number, - index?: number, - ) { - this.seriesId = seriesId; - this.timestamp = timestamp; - this.counter = counter; - this.index = index; - // TODO: will be removed once https://ably.atlassian.net/browse/DTP-1078 is implemented on the realtime - this.siteCode = this.seriesId.slice(0, 3); // site code is stored in the first 3 letters of the epoch, which is stored in the series id field - } - - /** - * Returns the string representation of the timeserial object. - * @returns The timeserial string. - */ - toString(): string { - return `${this.seriesId}@${this.timestamp.toString()}-${this.counter.toString()}${this.index ? `:${this.index.toString()}` : ''}`; - } - - /** - * Calculate the timeserial object from a timeserial string. - * - * @param timeserial The timeserial string to parse. - * @returns The parsed timeserial object. - * @throws {@link BaseClient.ErrorInfo | ErrorInfo} if timeserial is invalid. - */ - static calculateTimeserial(client: BaseClient, timeserial: string | null | undefined): Timeserial { - if (client.Utils.isNil(timeserial)) { - throw new client.ErrorInfo(`Invalid timeserial: ${timeserial}`, 50000, 500); - } - - const [seriesId, rest] = timeserial.split('@'); - if (!rest) { - throw new client.ErrorInfo(`Invalid timeserial: ${timeserial}`, 50000, 500); - } - - const [timestamp, counterAndIndex] = rest.split('-'); - if (!timestamp || !counterAndIndex) { - throw new client.ErrorInfo(`Invalid timeserial: ${timeserial}`, 50000, 500); - } - - const [counter, index] = counterAndIndex.split(':'); - if (!counter) { - throw new client.ErrorInfo(`Invalid timeserial: ${timeserial}`, 50000, 500); - } - - return new DefaultTimeserial( - client, - seriesId, - Number(timestamp), - Number(counter), - index ? Number(index) : undefined, - ); - } - - /** - * Returns a zero-value Timeserial `@0-0` - "earliest possible" timeserial. - * - * @returns The timeserial object. - */ - static zeroValueTimeserial(client: BaseClient): Timeserial { - return new DefaultTimeserial(client, '', 0, 0); // @0-0 - } - - /** - * Compares this timeserial to the supplied timeserial, returning a number indicating their relative order. - * @param timeserialToCompare The timeserial to compare against. Can be a string or a Timeserial object. - * @returns 0 if the timeserials are equal, <0 if the first timeserial is less than the second, >0 if the first timeserial is greater than the second. - * @throws {@link BaseClient.ErrorInfo | ErrorInfo} if comparison timeserial is invalid. - */ - private _timeserialCompare(timeserialToCompare: string | Timeserial): number { - const secondTimeserial = - typeof timeserialToCompare === 'string' - ? DefaultTimeserial.calculateTimeserial(this._client, timeserialToCompare) - : timeserialToCompare; - - // Compare the timestamp - const timestampDiff = this.timestamp - secondTimeserial.timestamp; - if (timestampDiff) { - return timestampDiff; - } - - // Compare the counter - const counterDiff = this.counter - secondTimeserial.counter; - if (counterDiff) { - return counterDiff; - } - - // Compare the seriesId lexicographically, but only if both seriesId exist - const seriesComparison = - this.seriesId && - secondTimeserial.seriesId && - this.seriesId !== secondTimeserial.seriesId && - (this.seriesId > secondTimeserial.seriesId ? 1 : -1); - if (seriesComparison) { - return seriesComparison; - } - - // Compare the index, if present - return this.index !== undefined && secondTimeserial.index !== undefined ? this.index - secondTimeserial.index : 0; - } - - /** - * Determines if this timeserial occurs logically before the given timeserial. - * - * @param timeserial The timeserial to compare against. Can be a string or a Timeserial object. - * @returns true if this timeserial precedes the given timeserial, in global order. - * @throws {@link BaseClient.ErrorInfo | ErrorInfo} if the given timeserial is invalid. - */ - before(timeserial: Timeserial | string): boolean { - return this._timeserialCompare(timeserial) < 0; - } - - /** - * Determines if this timeserial occurs logically after the given timeserial. - * - * @param timeserial The timeserial to compare against. Can be a string or a Timeserial object. - * @returns true if this timeserial follows the given timeserial, in global order. - * @throws {@link BaseClient.ErrorInfo | ErrorInfo} if the given timeserial is invalid. - */ - after(timeserial: Timeserial | string): boolean { - return this._timeserialCompare(timeserial) > 0; - } - - /** - * Determines if this timeserial is equal to the given timeserial. - * @param timeserial The timeserial to compare against. Can be a string or a Timeserial object. - * @returns true if this timeserial is equal to the given timeserial. - * @throws {@link BaseClient.ErrorInfo | ErrorInfo} if the given timeserial is invalid. - */ - equal(timeserial: Timeserial | string): boolean { - return this._timeserialCompare(timeserial) === 0; - } -} diff --git a/test/common/modules/live_objects_helper.js b/test/common/modules/live_objects_helper.js index 273a2f4999..a5c7a10f3f 100644 --- a/test/common/modules/live_objects_helper.js +++ b/test/common/modules/live_objects_helper.js @@ -220,9 +220,12 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb } stateOperationMessage(opts) { - const { channelName, serial, state } = opts; + const { channelName, serial, siteCode, state } = opts; - state?.forEach((x, i) => (x.serial = `${serial}:${i}`)); + state?.forEach((stateMessage, i) => { + stateMessage.serial = serial; + stateMessage.siteCode = siteCode; + }); return { action: 19, // STATE diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index ead316addc..01eb376fc7 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -31,20 +31,30 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], } function forScenarios(scenarios, testFn) { - // if there are scenarios marked as "only", run only them. - // otherwise go over every scenario - const onlyScenarios = scenarios.filter((x) => x.only === true); - const scenariosToRun = onlyScenarios.length > 0 ? onlyScenarios : scenarios; + for (const scenario of scenarios) { + const itFn = scenario.skip ? it.skip : scenario.only ? it.only : it; - for (const scenario of scenariosToRun) { - if (scenario.skip === true) { - continue; - } - - testFn(scenario); + itFn(scenario.description, async function () { + const helper = this.test.helper; + await testFn(helper, scenario); + }); } } + function lexicoTimeserial(seriesId, timestamp, counter, index) { + const paddedTimestamp = timestamp.toString().padStart(14, '0'); + const paddedCounter = counter.toString().padStart(3, '0'); + const paddedIndex = index != null ? index.toString().padStart(3, '0') : undefined; + + // Example: + // + // 01726585978590-001@abcdefghij:001 + // |____________| |_| |________| |_| + // | | | | + // timestamp counter seriesId idx + return `${paddedTimestamp}-${paddedCounter}@${seriesId}` + (paddedIndex ? `:${paddedIndex}` : ''); + } + describe('realtime/live_objects', function () { this.timeout(60 * 1000); @@ -91,7 +101,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // inject STATE message that should be ignored and not break anything without LiveObjects plugin await liveObjectsHelper.processStateOperationMessageOnChannel({ channel: testChannel, - serial: '@0-0', + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', state: [ liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'stringKey', data: { value: 'stringValue' } }), ], @@ -125,7 +136,12 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await liveObjectsHelper.processStateObjectMessageOnChannel({ channel: testChannel, syncSerial: 'serial:', - state: [liveObjectsHelper.mapObject({ objectId: 'root', siteTimeserials: { '000': '000@0-0' } })], + state: [ + liveObjectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + }), + ], }); const publishChannel = publishClient.channels.get('channel'); @@ -292,8 +308,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], state: [ liveObjectsHelper.mapObject({ objectId: 'root', - siteTimeserials: { '000': '000@0-0' }, - initialEntries: { key: { timeserial: '000@0-0', data: { value: 1 } } }, + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + initialEntries: { key: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 1 } } }, }), ], }); @@ -512,7 +528,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check no maps exist on root primitiveMapsFixtures.forEach((fixture) => { const key = fixture.name; - expect(root.get(key, `Check "${key}" key doesn't exist on root before applying MAP_CREATE ops`)).to.not + expect(root.get(key), `Check "${key}" key doesn't exist on root before applying MAP_CREATE ops`).to.not .exist; }); @@ -571,10 +587,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check map does not exist on root expect( - root.get( - withReferencesMapKey, - `Check "${withReferencesMapKey}" key doesn't exist on root before applying MAP_CREATE ops`, - ), + root.get(withReferencesMapKey), + `Check "${withReferencesMapKey}" key doesn't exist on root before applying MAP_CREATE ops`, ).to.not.exist; // create map with references. need to create referenced objects first to obtain their object ids @@ -653,28 +667,31 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // send a MAP_SET op first to create a zero-value map with forged site timeserials vector (from the op), and set it on a root. await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: 'bbb@1-0', + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', state: [liveObjectsHelper.mapSetOp({ objectId: mapId, key: 'foo', data: { value: 'bar' } })], }); await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: `aaa@${i}-0`, + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: mapId, data: { objectId: mapId } })], }); }), ); // inject operations with various timeserial values - for (const [i, serial] of [ - 'bbb@0-0', // existing site, earlier CGO, not applied - 'bbb@1-0', // existing site, same CGO, not applied - 'bbb@2-0', // existing site, later CGO, applied - 'aaa@0-0', // different site, earlier CGO, applied - 'ccc@9-0', // different site, later CGO, applied + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // existing site, earlier CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, same CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, later CGO, applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied ].entries()) { await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, serial, + siteCode, state: [ liveObjectsHelper.mapCreateOp({ objectId: mapIds[i], @@ -721,7 +738,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check root is empty before ops primitiveKeyData.forEach((keyData) => { expect( - root.get(keyData.key, `Check "${keyData.key}" key doesn't exist on root before applying MAP_SET ops`), + root.get(keyData.key), + `Check "${keyData.key}" key doesn't exist on root before applying MAP_SET ops`, ).to.not.exist; }); @@ -763,9 +781,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check no object ids are set on root expect( - root.get('keyToCounter', `Check "keyToCounter" key doesn't exist on root before applying MAP_SET ops`), + root.get('keyToCounter'), + `Check "keyToCounter" key doesn't exist on root before applying MAP_SET ops`, ).to.not.exist; - expect(root.get('keyToMap', `Check "keyToMap" key doesn't exist on root before applying MAP_SET ops`)).to + expect(root.get('keyToMap'), `Check "keyToMap" key doesn't exist on root before applying MAP_SET ops`).to .not.exist; // create new objects and set on root @@ -817,39 +836,42 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const mapId = liveObjectsHelper.fakeMapObjectId(); await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: 'bbb@1-0', + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', state: [ liveObjectsHelper.mapCreateOp({ objectId: mapId, entries: { - foo1: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo2: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo3: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo4: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo5: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo6: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo3: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo5: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo6: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, }, }), ], }); await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: 'aaa@0-0', + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'map', data: { objectId: mapId } })], }); // inject operations with various timeserial values - for (const [i, serial] of [ - 'bbb@0-0', // existing site, earlier site CGO, not applied - 'bbb@1-0', // existing site, same site CGO, not applied - 'bbb@2-0', // existing site, later site CGO, applied, site timeserials updated - 'bbb@2-0', // existing site, same site CGO (updated from last op), not applied - 'aaa@0-0', // different site, earlier entry CGO, not applied - 'ccc@9-0', // different site, later entry CGO, applied + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // existing site, earlier site CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, same site CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, later site CGO, applied, site timeserials updated + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, same site CGO (updated from last op), not applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier entry CGO, not applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later entry CGO, applied ].entries()) { await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, serial, + siteCode, state: [liveObjectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { value: 'baz' } })], }); } @@ -941,39 +963,42 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const mapId = liveObjectsHelper.fakeMapObjectId(); await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: 'bbb@1-0', + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', state: [ liveObjectsHelper.mapCreateOp({ objectId: mapId, entries: { - foo1: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo2: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo3: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo4: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo5: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo6: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, + foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo3: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo5: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo6: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, }, }), ], }); await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: 'aaa@0-0', + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'map', data: { objectId: mapId } })], }); // inject operations with various timeserial values - for (const [i, serial] of [ - 'bbb@0-0', // existing site, earlier site CGO, not applied - 'bbb@1-0', // existing site, same site CGO, not applied - 'bbb@2-0', // existing site, later site CGO, applied, site timeserials updated - 'bbb@2-0', // existing site, same site CGO (updated from last op), not applied - 'aaa@0-0', // different site, earlier entry CGO, not applied - 'ccc@9-0', // different site, later entry CGO, applied + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // existing site, earlier site CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, same site CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, later site CGO, applied, site timeserials updated + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, same site CGO (updated from last op), not applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier entry CGO, not applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later entry CGO, applied ].entries()) { await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, serial, + siteCode, state: [liveObjectsHelper.mapRemoveOp({ objectId: mapId, key: `foo${i + 1}` })], }); } @@ -1012,7 +1037,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check no counters exist on root countersFixtures.forEach((fixture) => { const key = fixture.name; - expect(root.get(key, `Check "${key}" key doesn't exist on root before applying COUNTER_CREATE ops`)).to + expect(root.get(key), `Check "${key}" key doesn't exist on root before applying COUNTER_CREATE ops`).to .not.exist; }); @@ -1069,12 +1094,14 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // send a COUNTER_INC op first to create a zero-value counter with forged site timeserials vector (from the op), and set it on a root. await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: 'bbb@1-0', + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], }); await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: `aaa@${i}-0`, + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', state: [ liveObjectsHelper.mapSetOp({ objectId: 'root', key: counterId, data: { objectId: counterId } }), ], @@ -1083,16 +1110,17 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ); // inject operations with various timeserial values - for (const [i, serial] of [ - 'bbb@0-0', // existing site, earlier CGO, not applied - 'bbb@1-0', // existing site, same CGO, not applied - 'bbb@2-0', // existing site, later CGO, applied - 'aaa@0-0', // different site, earlier CGO, applied - 'ccc@9-0', // different site, later CGO, applied + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // existing site, earlier CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, same CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, later CGO, applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied ].entries()) { await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, serial, + siteCode, state: [liveObjectsHelper.counterCreateOp({ objectId: counterIds[i], count: 10 })], }); } @@ -1186,27 +1214,30 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const counterId = liveObjectsHelper.fakeCounterObjectId(); await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: 'bbb@1-0', + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', state: [liveObjectsHelper.counterCreateOp({ objectId: counterId, count: 1 })], }); await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: 'aaa@0-0', + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'counter', data: { objectId: counterId } })], }); // inject operations with various timeserial values - for (const [i, serial] of [ - 'bbb@0-0', // +10 existing site, earlier CGO, not applied - 'bbb@1-0', // +100 existing site, same CGO, not applied - 'bbb@2-0', // +1000 existing site, later CGO, applied, site timeserials updated - 'bbb@2-0', // +10000 existing site, same CGO (updated from last op), not applied - 'aaa@0-0', // +100000 different site, earlier CGO, applied - 'ccc@9-0', // +1000000 different site, later CGO, applied + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // +10 existing site, earlier CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // +100 existing site, same CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // +1000 existing site, later CGO, applied, site timeserials updated + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // +10000 existing site, same CGO (updated from last op), not applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // +100000 different site, earlier CGO, applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // +1000000 different site, later CGO, applied ].entries()) { await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, serial, + siteCode, state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: Math.pow(10, i + 1) })], }); } @@ -1220,25 +1251,22 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, ]; - forScenarios(applyOperationsScenarios, (scenario) => - /** @nospec */ - it(scenario.description, async function () { - const helper = this.test.helper; - const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper); + /** @nospec */ + forScenarios(applyOperationsScenarios, async function (helper, scenario) { + const liveObjectsHelper = new LiveObjectsHelper(helper); + const client = RealtimeWithLiveObjects(helper); - await helper.monitorConnectionThenCloseAndFinish(async () => { - const channelName = scenario.description; - const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channelName = scenario.description; + const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; - await channel.attach(); - const root = await liveObjects.getRoot(); + await channel.attach(); + const root = await liveObjects.getRoot(); - await scenario.action({ root, liveObjectsHelper, channelName, channel }); - }, client); - }), - ); + await scenario.action({ root, liveObjectsHelper, channelName, channel }); + }, client); + }); const applyOperationsDuringSyncScenarios = [ { @@ -1257,7 +1285,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], primitiveKeyData.map((keyData) => liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: '@0-0', + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], }), ), @@ -1287,7 +1316,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], primitiveKeyData.map((keyData, i) => liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: `aaa@${i}-0`, + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], }), ), @@ -1329,10 +1359,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // inject operations, expect them to be discarded when sync with new sequence id starts await Promise.all( - primitiveKeyData.map((keyData) => + primitiveKeyData.map((keyData, i) => liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: '@0-0', + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], }), ), @@ -1347,7 +1378,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // inject another operation that should be applied when latest sync ends await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: '@0-0', + serial: lexicoTimeserial('bbb', 0, 0), + siteCode: 'bbb', state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { value: 'bar' } })], }); @@ -1391,34 +1423,34 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], liveObjectsHelper.mapObject({ objectId: mapId, siteTimeserials: { - bbb: 'bbb@2-0', - ccc: 'ccc@5-0', + bbb: lexicoTimeserial('bbb', 2, 0), + ccc: lexicoTimeserial('ccc', 5, 0), }, materialisedEntries: { - foo1: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo2: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo3: { timeserial: 'ccc@5-0', data: { value: 'bar' } }, - foo4: { timeserial: 'bbb@0-0', data: { value: 'bar' } }, - foo5: { timeserial: 'bbb@2-0', data: { value: 'bar' } }, - foo6: { timeserial: 'ccc@2-0', data: { value: 'bar' } }, - foo7: { timeserial: 'ccc@0-0', data: { value: 'bar' } }, - foo8: { timeserial: 'ccc@0-0', data: { value: 'bar' } }, + foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo3: { timeserial: lexicoTimeserial('ccc', 5, 0), data: { value: 'bar' } }, + foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo5: { timeserial: lexicoTimeserial('bbb', 2, 0), data: { value: 'bar' } }, + foo6: { timeserial: lexicoTimeserial('ccc', 2, 0), data: { value: 'bar' } }, + foo7: { timeserial: lexicoTimeserial('ccc', 0, 0), data: { value: 'bar' } }, + foo8: { timeserial: lexicoTimeserial('ccc', 0, 0), data: { value: 'bar' } }, }, }), liveObjectsHelper.counterObject({ objectId: counterId, siteTimeserials: { - bbb: 'bbb@1-0', + bbb: lexicoTimeserial('bbb', 1, 0), }, initialCount: 1, }), // add objects to the root so they're discoverable in the state tree liveObjectsHelper.mapObject({ objectId: 'root', - siteTimeserials: { '000': '000@0-0' }, + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, initialEntries: { - map: { timeserial: '000@0-0', data: { objectId: mapId } }, - counter: { timeserial: '000@0-0', data: { objectId: counterId } }, + map: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: mapId } }, + counter: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId } }, }, }), ], @@ -1426,37 +1458,39 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // inject operations with various timeserial values // Map: - for (const [i, serial] of [ - 'bbb@1-0', // existing site, earlier site CGO, not applied - 'bbb@2-0', // existing site, same site CGO, not applied - 'bbb@3-0', // existing site, later site CGO, earlier entry CGO, not applied but site timeserial updated + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, earlier site CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, same site CGO, not applied + { serial: lexicoTimeserial('bbb', 3, 0), siteCode: 'bbb' }, // existing site, later site CGO, earlier entry CGO, not applied but site timeserial updated // message with later site CGO, same entry CGO case is not possible, as timeserial from entry would be set for the corresponding site code or be less than that - 'bbb@3-0', // existing site, same site CGO (updated from last op), later entry CGO, not applied - 'bbb@4-0', // existing site, later site CGO, later entry CGO, applied - 'aaa@1-0', // different site, earlier entry CGO, not applied but site timeserial updated - 'aaa@1-0', // different site, same site CGO (updated from last op), later entry CGO, not applied + { serial: lexicoTimeserial('bbb', 3, 0), siteCode: 'bbb' }, // existing site, same site CGO (updated from last op), later entry CGO, not applied + { serial: lexicoTimeserial('bbb', 4, 0), siteCode: 'bbb' }, // existing site, later site CGO, later entry CGO, applied + { serial: lexicoTimeserial('aaa', 1, 0), siteCode: 'aaa' }, // different site, earlier entry CGO, not applied but site timeserial updated + { serial: lexicoTimeserial('aaa', 1, 0), siteCode: 'aaa' }, // different site, same site CGO (updated from last op), later entry CGO, not applied // different site with matching entry CGO case is not possible, as matching entry timeserial means that that timeserial is in the site timeserials vector - 'ddd@1-0', // different site, later entry CGO, applied + { serial: lexicoTimeserial('ddd', 1, 0), siteCode: 'ddd' }, // different site, later entry CGO, applied ].entries()) { await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, serial, + siteCode, state: [liveObjectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { value: 'baz' } })], }); } // Counter: - for (const [i, serial] of [ - 'bbb@0-0', // +10 existing site, earlier CGO, not applied - 'bbb@1-0', // +100 existing site, same CGO, not applied - 'bbb@2-0', // +1000 existing site, later CGO, applied, site timeserials updated - 'bbb@2-0', // +10000 existing site, same CGO (updated from last op), not applied - 'aaa@0-0', // +100000 different site, earlier CGO, applied - 'ccc@9-0', // +1000000 different site, later CGO, applied + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // +10 existing site, earlier CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // +100 existing site, same CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // +1000 existing site, later CGO, applied, site timeserials updated + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // +10000 existing site, same CGO (updated from last op), not applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // +100000 different site, earlier CGO, applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // +1000000 different site, later CGO, applied ].entries()) { await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, serial, + siteCode, state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: Math.pow(10, i + 1) })], }); } @@ -1510,7 +1544,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], primitiveKeyData.map((keyData, i) => liveObjectsHelper.processStateOperationMessageOnChannel({ channel, - serial: `aaa@${i}-0`, + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], }), ), @@ -1554,27 +1589,24 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, ]; - forScenarios(applyOperationsDuringSyncScenarios, (scenario) => - /** @nospec */ - it(scenario.description, async function () { - const helper = this.test.helper; - const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper); + /** @nospec */ + forScenarios(applyOperationsDuringSyncScenarios, async function (helper, scenario) { + const liveObjectsHelper = new LiveObjectsHelper(helper); + const client = RealtimeWithLiveObjects(helper); - await helper.monitorConnectionThenCloseAndFinish(async () => { - const channelName = scenario.description; - const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channelName = scenario.description; + const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; - await channel.attach(); - // wait for getRoot() to resolve so the initial SYNC sequence is completed, - // as we're going to initiate a new one to test applying operations during SYNC sequence. - const root = await liveObjects.getRoot(); + await channel.attach(); + // wait for getRoot() to resolve so the initial SYNC sequence is completed, + // as we're going to initiate a new one to test applying operations during SYNC sequence. + const root = await liveObjects.getRoot(); - await scenario.action({ root, liveObjectsHelper, channelName, channel }); - }, client); - }), - ); + await scenario.action({ root, liveObjectsHelper, channelName, channel }); + }, client); + }); const subscriptionCallbacksScenarios = [ { @@ -2043,49 +2075,46 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, ]; - forScenarios(subscriptionCallbacksScenarios, (scenario) => - /** @nospec */ - it(scenario.description, async function () { - const helper = this.test.helper; - const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper); + /** @nospec */ + forScenarios(subscriptionCallbacksScenarios, async function (helper, scenario) { + const liveObjectsHelper = new LiveObjectsHelper(helper); + const client = RealtimeWithLiveObjects(helper); - await helper.monitorConnectionThenCloseAndFinish(async () => { - const channelName = scenario.description; - const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channelName = scenario.description; + const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; - await channel.attach(); - const root = await liveObjects.getRoot(); + await channel.attach(); + const root = await liveObjects.getRoot(); - const sampleMapKey = 'sampleMap'; - const sampleCounterKey = 'sampleCounter'; + const sampleMapKey = 'sampleMap'; + const sampleCounterKey = 'sampleCounter'; - // prepare map and counter objects for use by the scenario - const { objectId: sampleMapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { - mapObjectId: 'root', - key: sampleMapKey, - createOp: liveObjectsHelper.mapCreateOp(), - }); - const { objectId: sampleCounterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { - mapObjectId: 'root', - key: sampleCounterKey, - createOp: liveObjectsHelper.counterCreateOp(), - }); + // prepare map and counter objects for use by the scenario + const { objectId: sampleMapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: sampleMapKey, + createOp: liveObjectsHelper.mapCreateOp(), + }); + const { objectId: sampleCounterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: sampleCounterKey, + createOp: liveObjectsHelper.counterCreateOp(), + }); - await scenario.action({ - root, - liveObjectsHelper, - channelName, - channel, - sampleMapKey, - sampleMapObjectId, - sampleCounterKey, - sampleCounterObjectId, - }); - }, client); - }), - ); + await scenario.action({ + root, + liveObjectsHelper, + channelName, + channel, + sampleMapKey, + sampleMapObjectId, + sampleCounterKey, + sampleCounterObjectId, + }); + }, client); + }); }); /** @nospec */ From 7927b24bdd70b3a910ff9fffbd5aeb6167c136bc Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 4 Dec 2024 05:43:32 +0000 Subject: [PATCH 079/166] Move `applyStateMessages` to LiveObjects class --- src/plugins/liveobjects/liveobjects.ts | 47 ++++++++++++++++++++-- src/plugins/liveobjects/liveobjectspool.ts | 45 --------------------- 2 files changed, 44 insertions(+), 48 deletions(-) diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 75b743d5b0..aa064de78a 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -6,7 +6,7 @@ import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject, LiveObjectUpdate } from './liveobject'; import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; -import { StateMessage } from './statemessage'; +import { StateMessage, StateOperationAction } from './statemessage'; import { SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; enum LiveObjectsEvents { @@ -101,7 +101,7 @@ export class LiveObjects { return; } - this._liveObjectsPool.applyStateMessages(stateMessages); + this._applyStateMessages(stateMessages); } /** @@ -159,7 +159,7 @@ export class LiveObjects { this._applySync(); // should apply buffered state operations after we applied the SYNC data. // can use regular state messages application logic - this._liveObjectsPool.applyStateMessages(this._bufferedStateOperations); + this._applyStateMessages(this._bufferedStateOperations); this._bufferedStateOperations = []; this._syncLiveObjectsDataPool.reset(); @@ -232,4 +232,45 @@ export class LiveObjects { // call subscription callbacks for all updated existing objects existingObjectUpdates.forEach(({ object, update }) => object.notifyUpdated(update)); } + + private _applyStateMessages(stateMessages: StateMessage[]): void { + for (const stateMessage of stateMessages) { + if (!stateMessage.operation) { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MAJOR, + 'LiveObjects._applyStateMessages()', + `state operation message is received without 'operation' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + ); + continue; + } + + const stateOperation = stateMessage.operation; + + switch (stateOperation.action) { + case StateOperationAction.MAP_CREATE: + case StateOperationAction.COUNTER_CREATE: + case StateOperationAction.MAP_SET: + case StateOperationAction.MAP_REMOVE: + case StateOperationAction.COUNTER_INC: + // we can receive an op for an object id we don't have yet in the pool. instead of buffering such operations, + // we can create a zero-value object for the provided object id and apply the operation to that zero-value object. + // this also means that all objects are capable of applying the corresponding *_CREATE ops on themselves, + // since they need to be able to eventually initialize themselves from that *_CREATE op. + // so to simplify operations handling, we always try to create a zero-value object in the pool first, + // and then we can always apply the operation on the existing object in the pool. + this._liveObjectsPool.createZeroValueObjectIfNotExists(stateOperation.objectId); + this._liveObjectsPool.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); + break; + + default: + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MAJOR, + 'LiveObjects._applyStateMessages()', + `received unsupported action in state operation message: ${stateOperation.action}, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + ); + } + } + } } diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index 2c57f1084e..eb42d47b4e 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -1,11 +1,9 @@ import type BaseClient from 'common/lib/client/baseclient'; -import type RealtimeChannel from 'common/lib/client/realtimechannel'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; import { LiveObjects } from './liveobjects'; import { ObjectId } from './objectid'; -import { StateMessage, StateOperationAction } from './statemessage'; export const ROOT_OBJECT_ID = 'root'; @@ -14,12 +12,10 @@ export const ROOT_OBJECT_ID = 'root'; */ export class LiveObjectsPool { private _client: BaseClient; - private _channel: RealtimeChannel; private _pool: Map; constructor(private _liveObjects: LiveObjects) { this._client = this._liveObjects.getClient(); - this._channel = this._liveObjects.getChannel(); this._pool = this._getInitialPool(); } @@ -66,47 +62,6 @@ export class LiveObjectsPool { this.set(objectId, zeroValueObject); } - applyStateMessages(stateMessages: StateMessage[]): void { - for (const stateMessage of stateMessages) { - if (!stateMessage.operation) { - this._client.Logger.logAction( - this._client.logger, - this._client.Logger.LOG_MAJOR, - 'LiveObjects.LiveObjectsPool.applyStateMessages()', - `state operation message is received without 'operation' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, - ); - continue; - } - - const stateOperation = stateMessage.operation; - - switch (stateOperation.action) { - case StateOperationAction.MAP_CREATE: - case StateOperationAction.COUNTER_CREATE: - case StateOperationAction.MAP_SET: - case StateOperationAction.MAP_REMOVE: - case StateOperationAction.COUNTER_INC: - // we can receive an op for an object id we don't have yet in the pool. instead of buffering such operations, - // we can create a zero-value object for the provided object id and apply the operation to that zero-value object. - // this also means that all objects are capable of applying the corresponding *_CREATE ops on themselves, - // since they need to be able to eventually initialize themselves from that *_CREATE op. - // so to simplify operations handling, we always try to create a zero-value object in the pool first, - // and then we can always apply the operation on the existing object in the pool. - this.createZeroValueObjectIfNotExists(stateOperation.objectId); - this.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); - break; - - default: - this._client.Logger.logAction( - this._client.logger, - this._client.Logger.LOG_MAJOR, - 'LiveObjects.LiveObjectsPool.applyStateMessages()', - `received unsupported action in state operation message: ${stateOperation.action}, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, - ); - } - } - } - private _getInitialPool(): Map { const pool = new Map(); const root = LiveMap.zeroValue(this._liveObjects, ROOT_OBJECT_ID); From c47b8c42f49406bc8f0b88a3aa94fa8ec3840951 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 5 Dec 2024 06:43:26 +0000 Subject: [PATCH 080/166] Handle `StateObject.tombstone` and `OBJECT_DELETE` messages Tombstoned objects clear their underlying data by setting it to a zero value: 0 for a counter object, empty map for a map. No state operations can be applied on a tombstoned object. Tombstoned objects are not surfaced to the end users. When deleted, object triggers a subscription callback with cleared data. Resolves DTP-986 --- ably.d.ts | 6 +- src/plugins/liveobjects/livecounter.ts | 39 +- src/plugins/liveobjects/livemap.ts | 80 ++- src/plugins/liveobjects/liveobject.ts | 32 +- src/plugins/liveobjects/liveobjects.ts | 5 +- src/plugins/liveobjects/statemessage.ts | 3 + test/common/modules/live_objects_helper.js | 19 +- .../browser/template/src/ably.config.d.ts | 4 +- .../browser/template/src/index-liveobjects.ts | 4 +- test/realtime/live_objects.test.js | 526 +++++++++++++++++- 10 files changed, 653 insertions(+), 65 deletions(-) diff --git a/ably.d.ts b/ably.d.ts index 5d1aa1f825..cd60057e74 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -2103,12 +2103,12 @@ export type DefaultRoot = */ export declare interface LiveMap extends LiveObject { /** - * Returns the value associated with a given key. Returns `undefined` if the key doesn't exist in a map. + * Returns the value associated with a given key. Returns `undefined` if the key doesn't exist in a map or if the associated {@link LiveObject} has been deleted. * * @param key - The key to retrieve the value for. - * @returns A {@link LiveObject}, a primitive type (string, number, boolean, or binary data) or `undefined` if the key doesn't exist in a map. + * @returns A {@link LiveObject}, a primitive type (string, number, boolean, or binary data) or `undefined` if the key doesn't exist in a map or the associated {@link LiveObject} has been deleted. */ - get(key: TKey): T[TKey]; + get(key: TKey): T[TKey] extends StateValue ? T[TKey] : T[TKey] | undefined; /** * Returns the number of key/value pairs in the map. diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index dc144c3259..c96d59afe4 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -63,6 +63,11 @@ export class LiveCounter extends LiveObject // as it's important to mark that the op was processed by the object this._siteTimeserials[opSiteCode] = opOriginTimeserial; + if (this.isTombstoned()) { + // this object is tombstoned so the operation cannot be applied + return; + } + let update: LiveCounterUpdate | LiveObjectUpdateNoop; switch (op.action) { case StateOperationAction.COUNTER_CREATE: @@ -79,6 +84,10 @@ export class LiveCounter extends LiveObject } break; + case StateOperationAction.OBJECT_DELETE: + update = this._applyObjectDelete(); + break; + default: throw new this._client.ErrorInfo( `Invalid ${op.action} op for LiveCounter objectId=${this.getObjectId()}`, @@ -93,7 +102,7 @@ export class LiveCounter extends LiveObject /** * @internal */ - overrideWithStateObject(stateObject: StateObject): LiveCounterUpdate { + overrideWithStateObject(stateObject: StateObject): LiveCounterUpdate | LiveObjectUpdateNoop { if (stateObject.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( `Invalid state object: state object objectId=${stateObject.objectId}; LiveCounter objectId=${this.getObjectId()}`, @@ -121,16 +130,30 @@ export class LiveCounter extends LiveObject } } - const previousDataRef = this._dataRef; - // override all relevant data for this object with data from the state object - this._createOperationIsMerged = false; - this._dataRef = { data: stateObject.counter?.count ?? 0 }; - // should default to empty map if site timeserials do not exist on the state object, so that any future operation can be applied to this object + // object's site timeserials are still updated even if it is tombstoned, so always use the site timeserials received from the op. + // should default to empty map if site timeserials do not exist on the state object, so that any future operation may be applied to this object. this._siteTimeserials = stateObject.siteTimeserials ?? {}; - if (!this._client.Utils.isNil(stateObject.createOp)) { - this._mergeInitialDataFromCreateOperation(stateObject.createOp); + + if (this.isTombstoned()) { + // this object is tombstoned. this is a terminal state which can't be overriden. skip the rest of state object message processing + return { noop: true }; + } + + const previousDataRef = this._dataRef; + if (stateObject.tombstone) { + // tombstone this object and ignore the data from the state object message + this.tombstone(); + } else { + // override data for this object with data from the state object + this._createOperationIsMerged = false; + this._dataRef = { data: stateObject.counter?.count ?? 0 }; + if (!this._client.Utils.isNil(stateObject.createOp)) { + this._mergeInitialDataFromCreateOperation(stateObject.createOp); + } } + // if object got tombstoned, the update object will include all data that got cleared. + // otherwise it is a diff between previous value and new value from state object. return this._updateFromDataDiff(previousDataRef, this._dataRef); } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 1f3729c5f0..7271beb5b5 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -77,13 +77,15 @@ export class LiveMap extends LiveObject(key: TKey): T[TKey] { + get(key: TKey): T[TKey] extends StateValue ? T[TKey] : T[TKey] | undefined { const element = this._dataRef.data.get(key); if (element === undefined) { @@ -94,14 +96,26 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject; protected _createOperationIsMerged: boolean; + private _tombstone: boolean; protected constructor( protected _liveObjects: LiveObjects, @@ -46,11 +47,12 @@ export abstract class LiveObject< ) { this._client = this._liveObjects.getClient(); this._eventEmitter = new this._client.EventEmitter(this._client.logger); - this._dataRef = this._getZeroValueData(); - this._createOperationIsMerged = false; this._objectId = objectId; + this._dataRef = this._getZeroValueData(); // use empty timeserials vector by default, so any future operation can be applied to this object this._siteTimeserials = {}; + this._createOperationIsMerged = false; + this._tombstone = false; } subscribe(listener: (update: TUpdate) => void): SubscribeResponse { @@ -99,6 +101,24 @@ export abstract class LiveObject< this._eventEmitter.emit(LiveObjectEvents.Updated, update); } + /** + * Clears the object's state, cancels any buffered operations and sets the tombstone flag to `true`. + * + * @internal + */ + tombstone(): void { + this._tombstone = true; + this._dataRef = this._getZeroValueData(); + // TODO: emit "deleted" event so that end users get notified about this object getting deleted + } + + /** + * @internal + */ + isTombstoned(): boolean { + return this._tombstone; + } + /** * Returns true if the given origin timeserial indicates that the operation to which it belongs should be applied to the object. * @@ -118,6 +138,12 @@ export abstract class LiveObject< return !siteTimeserial || opOriginTimeserial > siteTimeserial; } + protected _applyObjectDelete(): TUpdate { + const previousDataRef = this._dataRef; + this.tombstone(); + return this._updateFromDataDiff(previousDataRef, this._dataRef); + } + private _createObjectId(): string { // TODO: implement object id generation based on live object type and initial value return Math.random().toString().substring(2); @@ -141,7 +167,7 @@ export abstract class LiveObject< * * @internal */ - abstract overrideWithStateObject(stateObject: StateObject): TUpdate; + abstract overrideWithStateObject(stateObject: StateObject): TUpdate | LiveObjectUpdateNoop; protected abstract _getZeroValueData(): TData; /** * Calculate the update object based on the current Live Object data and incoming new data. diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index aa064de78a..6ba3a0f2c7 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -4,7 +4,7 @@ import type EventEmitter from 'common/lib/util/eventemitter'; import type * as API from '../../../ably'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; -import { LiveObject, LiveObjectUpdate } from './liveobject'; +import { LiveObject, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage, StateOperationAction } from './statemessage'; import { SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; @@ -193,7 +193,7 @@ export class LiveObjects { } const receivedObjectIds = new Set(); - const existingObjectUpdates: { object: LiveObject; update: LiveObjectUpdate }[] = []; + const existingObjectUpdates: { object: LiveObject; update: LiveObjectUpdate | LiveObjectUpdateNoop }[] = []; for (const [objectId, entry] of this._syncLiveObjectsDataPool.entries()) { receivedObjectIds.add(objectId); @@ -253,6 +253,7 @@ export class LiveObjects { case StateOperationAction.MAP_SET: case StateOperationAction.MAP_REMOVE: case StateOperationAction.COUNTER_INC: + case StateOperationAction.OBJECT_DELETE: // we can receive an op for an object id we don't have yet in the pool. instead of buffering such operations, // we can create a zero-value object for the provided object id and apply the operation to that zero-value object. // this also means that all objects are capable of applying the corresponding *_CREATE ops on themselves, diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 74b3856300..5dd409811d 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -8,6 +8,7 @@ export enum StateOperationAction { MAP_REMOVE = 2, COUNTER_CREATE = 3, COUNTER_INC = 4, + OBJECT_DELETE = 5, } export enum MapSemantics { @@ -106,6 +107,8 @@ export interface StateObject { objectId: string; /** A vector of origin timeserials keyed by site code of the last operation that was applied to this state object. */ siteTimeserials: Record; + /** True if the object has been tombstoned. */ + tombstone: boolean; /** * The operation that created the state object. * diff --git a/test/common/modules/live_objects_helper.js b/test/common/modules/live_objects_helper.js index a5c7a10f3f..b5e42f79de 100644 --- a/test/common/modules/live_objects_helper.js +++ b/test/common/modules/live_objects_helper.js @@ -12,6 +12,7 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb MAP_REMOVE: 2, COUNTER_CREATE: 3, COUNTER_INC: 4, + OBJECT_DELETE: 5, }; function nonce() { @@ -180,12 +181,25 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb return op; } + objectDeleteOp(opts) { + const { objectId } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.OBJECT_DELETE, + objectId, + }, + }; + + return op; + } + mapObject(opts) { - const { objectId, siteTimeserials, initialEntries, materialisedEntries } = opts; + const { objectId, siteTimeserials, initialEntries, materialisedEntries, tombstone } = opts; const obj = { object: { objectId, siteTimeserials, + tombstone: tombstone === true, map: { semantics: 0, entries: materialisedEntries, @@ -201,11 +215,12 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb } counterObject(opts) { - const { objectId, siteTimeserials, initialCount, materialisedCount } = opts; + const { objectId, siteTimeserials, initialCount, materialisedCount, tombstone } = opts; const obj = { object: { objectId, siteTimeserials, + tombstone: tombstone === true, counter: { count: materialisedCount, }, diff --git a/test/package/browser/template/src/ably.config.d.ts b/test/package/browser/template/src/ably.config.d.ts index e5bca7718f..3b3c69ddb1 100644 --- a/test/package/browser/template/src/ably.config.d.ts +++ b/test/package/browser/template/src/ably.config.d.ts @@ -5,13 +5,13 @@ type CustomRoot = { stringKey: string; booleanKey: boolean; couldBeUndefined?: string; - mapKey?: LiveMap<{ + mapKey: LiveMap<{ foo: 'bar'; nestedMap?: LiveMap<{ baz: 'qux'; }>; }>; - counterKey?: LiveCounter; + counterKey: LiveCounter; }; declare global { diff --git a/test/package/browser/template/src/index-liveobjects.ts b/test/package/browser/template/src/index-liveobjects.ts index 1cd27b0217..00763d46be 100644 --- a/test/package/browser/template/src/index-liveobjects.ts +++ b/test/package/browser/template/src/index-liveobjects.ts @@ -30,8 +30,10 @@ globalThis.testAblyPackage = async function () { const aBoolean: boolean = root.get('booleanKey'); const couldBeUndefined: string | undefined = root.get('couldBeUndefined'); // live objects on a root: + // LiveMap.get can still return undefined for LiveObject typed properties even if custom typings have them as non-optional. + // objects can be tombstoned and result in the undefined value const counter: Ably.LiveCounter | undefined = root.get('counterKey'); - const map: LiveObjectsTypes['root']['mapKey'] = root.get('mapKey'); + const map: LiveObjectsTypes['root']['mapKey'] | undefined = root.get('mapKey'); // check string literal types works // need to use nullish coalescing as we didn't actually create any data on the root, // so the next calls would fail. we only need to check that TypeScript types work diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 01eb376fc7..22164ef795 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -515,6 +515,162 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { name: 'maxSafeIntegerCounter', count: Number.MAX_SAFE_INTEGER }, { name: 'negativeMaxSafeIntegerCounter', count: -Number.MAX_SAFE_INTEGER }, ]; + + const stateSyncSequenceScanarios = [ + { + description: 'STATE_SYNC sequence with state object "tombstone" property creates tombstoned object', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + const mapId = liveObjectsHelper.fakeMapObjectId(); + const counterId = liveObjectsHelper.fakeCounterObjectId(); + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately + // add state objects with tombstone=true + state: [ + liveObjectsHelper.mapObject({ + objectId: mapId, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialEntries: {}, + }), + liveObjectsHelper.counterObject({ + objectId: counterId, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialCount: 1, + }), + liveObjectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + initialEntries: { + map: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: mapId } }, + counter: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId } }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'bar' } }, + }, + }), + ], + }); + + expect( + root.get('map'), + 'Check map does not exist on root after STATE_SYNC with "tombstone=true" for a map object', + ).to.not.exist; + expect( + root.get('counter'), + 'Check counter does not exist on root after STATE_SYNC with "tombstone=true" for a counter object', + ).to.not.exist; + // control check that STATE_SYNC was applied at all + expect(root.get('foo'), 'Check property exists on root after STATE_SYNC').to.exist; + }, + }, + + { + description: 'STATE_SYNC sequence with state object "tombstone" property deletes existing object', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, channel } = ctx; + + const { objectId: counterId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: liveObjectsHelper.counterCreateOp({ count: 1 }), + }); + + expect(root.get('counter'), 'Check counter exists on root before STATE_SYNC sequence with "tombstone=true"') + .to.exist; + + // inject a STATE_SYNC sequence where a counter is now tombstoned + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately + state: [ + liveObjectsHelper.counterObject({ + objectId: counterId, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialCount: 1, + }), + liveObjectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + initialEntries: { + counter: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId } }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'bar' } }, + }, + }), + ], + }); + + expect( + root.get('counter'), + 'Check counter does not exist on root after STATE_SYNC with "tombstone=true" for an existing counter object', + ).to.not.exist; + // control check that STATE_SYNC was applied at all + expect(root.get('foo'), 'Check property exists on root after STATE_SYNC').to.exist; + }, + }, + + { + description: + 'STATE_SYNC sequence with state object "tombstone" property triggers subscription callback for existing object', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, channel } = ctx; + + const { objectId: counterId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: liveObjectsHelper.counterCreateOp({ count: 1 }), + }); + + const counterSubPromise = new Promise((resolve, reject) => + root.get('counter').subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { inc: -1 } }, + 'Check counter subscription callback is called with an expected update object after STATE_SYNC sequence with "tombstone=true"', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + + // inject a STATE_SYNC sequence where a counter is now tombstoned + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately + state: [ + liveObjectsHelper.counterObject({ + objectId: counterId, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialCount: 1, + }), + liveObjectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + initialEntries: { + counter: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId } }, + }, + }), + ], + }); + + await counterSubPromise; + }, + }, + ]; + const applyOperationsScenarios = [ { description: 'can apply MAP_CREATE with primitives state operation messages', @@ -1249,24 +1405,337 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ); }, }, - ]; - /** @nospec */ - forScenarios(applyOperationsScenarios, async function (helper, scenario) { - const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper); + { + description: 'can apply OBJECT_DELETE state operation messages', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, channel } = ctx; - await helper.monitorConnectionThenCloseAndFinish(async () => { - const channelName = scenario.description; - const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + // create initial objects and set on root + const { objectId: mapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: liveObjectsHelper.mapCreateOp(), + }); + const { objectId: counterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: liveObjectsHelper.counterCreateOp(), + }); - await channel.attach(); - const root = await liveObjects.getRoot(); + expect(root.get('map'), 'Check map exists on root before OBJECT_DELETE').to.exist; + expect(root.get('counter'), 'Check counter exists on root before OBJECT_DELETE').to.exist; - await scenario.action({ root, liveObjectsHelper, channelName, channel }); - }, client); - }); + // inject OBJECT_DELETE + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.objectDeleteOp({ objectId: mapObjectId })], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.objectDeleteOp({ objectId: counterObjectId })], + }); + + expect(root.get('map'), 'Check map is not accessible on root after OBJECT_DELETE').to.not.exist; + expect(root.get('counter'), 'Check counter is not accessible on root after OBJECT_DELETE').to.not.exist; + }, + }, + + { + description: 'OBJECT_DELETE for unknown object id creates zero-value tombstoned object', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + const counterId = liveObjectsHelper.fakeCounterObjectId(); + // inject OBJECT_DELETE. should create a zero-value tombstoned object which can't be modified + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.objectDeleteOp({ objectId: counterId })], + }); + + // try to create and set tombstoned object on root + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 0, 0), + siteCode: 'bbb', + state: [liveObjectsHelper.counterCreateOp({ objectId: counterId })], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', + state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'counter', data: { objectId: counterId } })], + }); + + expect(root.get('counter'), 'Check counter is not accessible on root').to.not.exist; + }, + }, + + { + description: + 'OBJECT_DELETE state operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, liveObjectsHelper, channel } = ctx; + + // need to use multiple objects as OBJECT_DELETE op can only be applied once to an object + const counterIds = [ + liveObjectsHelper.fakeCounterObjectId(), + liveObjectsHelper.fakeCounterObjectId(), + liveObjectsHelper.fakeCounterObjectId(), + liveObjectsHelper.fakeCounterObjectId(), + liveObjectsHelper.fakeCounterObjectId(), + ]; + await Promise.all( + counterIds.map(async (counterId, i) => { + // create objects and set them on root with forged timeserials + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', + state: [liveObjectsHelper.counterCreateOp({ objectId: counterId })], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', + state: [ + liveObjectsHelper.mapSetOp({ objectId: 'root', key: counterId, data: { objectId: counterId } }), + ], + }); + }), + ); + + // inject OBJECT_DELETE operations with various timeserial values + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // existing site, earlier CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, same CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, later CGO, applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied + ].entries()) { + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial, + siteCode, + state: [liveObjectsHelper.objectDeleteOp({ objectId: counterIds[i] })], + }); + } + + // check only operations with correct timeserials were applied + const expectedCounters = [ + { exists: true }, + { exists: true }, + { exists: false }, // OBJECT_DELETE applied + { exists: false }, // OBJECT_DELETE applied + { exists: false }, // OBJECT_DELETE applied + ]; + + for (const [i, counterId] of counterIds.entries()) { + const { exists } = expectedCounters[i]; + + if (exists) { + expect( + root.get(counterId), + `Check counter #${i + 1} exists on root as OBJECT_DELETE op was not applied`, + ).to.exist; + } else { + expect( + root.get(counterId), + `Check counter #${i + 1} does not exist on root as OBJECT_DELETE op was applied`, + ).to.not.exist; + } + } + }, + }, + + { + description: 'OBJECT_DELETE triggers subscription callback with deleted data', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, channel } = ctx; + + // create initial objects and set on root + const { objectId: mapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: liveObjectsHelper.mapCreateOp({ + entries: { + foo: { data: { value: 'bar' } }, + baz: { data: { value: 1 } }, + }, + }), + }); + const { objectId: counterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: liveObjectsHelper.counterCreateOp({ count: 1 }), + }); + + const mapSubPromise = new Promise((resolve, reject) => + root.get('map').subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { foo: 'removed', baz: 'removed' } }, + 'Check map subscription callback is called with an expected update object after OBJECT_DELETE operation', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + const counterSubPromise = new Promise((resolve, reject) => + root.get('counter').subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { inc: -1 } }, + 'Check counter subscription callback is called with an expected update object after OBJECT_DELETE operation', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + + // inject OBJECT_DELETE + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.objectDeleteOp({ objectId: mapObjectId })], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.objectDeleteOp({ objectId: counterObjectId })], + }); + + await Promise.all([mapSubPromise, counterSubPromise]); + }, + }, + + { + description: 'MAP_SET with reference to a tombstoned object results in undefined value on key', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, channel } = ctx; + + // create initial objects and set on root + const { objectId: counterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'foo', + createOp: liveObjectsHelper.counterCreateOp(), + }); + + expect(root.get('foo'), 'Check counter exists on root before OBJECT_DELETE').to.exist; + + // inject OBJECT_DELETE + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.objectDeleteOp({ objectId: counterObjectId })], + }); + + // set tombstoned counter to another key on root + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [ + liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'bar', data: { objectId: counterObjectId } }), + ], + }); + + expect(root.get('bar'), 'Check counter is not accessible on new key in root after OBJECT_DELETE').to.not + .exist; + }, + }, + + { + description: 'state operation message on a tombstoned object does not revive it', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, channel } = ctx; + + // create initial objects and set on root + const { objectId: mapId1 } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map1', + createOp: liveObjectsHelper.mapCreateOp(), + }); + const { objectId: mapId2 } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map2', + createOp: liveObjectsHelper.mapCreateOp({ entries: { foo: { data: { value: 'bar' } } } }), + }); + const { objectId: counterId1 } = await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter1', + createOp: liveObjectsHelper.counterCreateOp(), + }); + + expect(root.get('map1'), 'Check map1 exists on root before OBJECT_DELETE').to.exist; + expect(root.get('map2'), 'Check map2 exists on root before OBJECT_DELETE').to.exist; + expect(root.get('counter1'), 'Check counter1 exists on root before OBJECT_DELETE').to.exist; + + // inject OBJECT_DELETE + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.objectDeleteOp({ objectId: mapId1 })], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.objectDeleteOp({ objectId: mapId2 })], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 2, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.objectDeleteOp({ objectId: counterId1 })], + }); + + // inject state ops on tombstoned objects + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 3, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.mapSetOp({ objectId: mapId1, key: 'baz', data: { value: 'qux' } })], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 4, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.mapRemoveOp({ objectId: mapId2, key: 'foo' })], + }); + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 5, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.counterIncOp({ objectId: counterId1, amount: 1 })], + }); + + // objects should still be deleted + expect(root.get('map1'), 'Check map1 does not exist on root after OBJECT_DELETE and another state op').to + .not.exist; + expect(root.get('map2'), 'Check map2 does not exist on root after OBJECT_DELETE and another state op').to + .not.exist; + expect( + root.get('counter1'), + 'Check counter1 does not exist on root after OBJECT_DELETE and another state op', + ).to.not.exist; + }, + }, + ]; const applyOperationsDuringSyncScenarios = [ { @@ -1590,23 +2059,24 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ]; /** @nospec */ - forScenarios(applyOperationsDuringSyncScenarios, async function (helper, scenario) { - const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper); + forScenarios( + [...stateSyncSequenceScanarios, ...applyOperationsScenarios, ...applyOperationsDuringSyncScenarios], + async function (helper, scenario) { + const liveObjectsHelper = new LiveObjectsHelper(helper); + const client = RealtimeWithLiveObjects(helper); - await helper.monitorConnectionThenCloseAndFinish(async () => { - const channelName = scenario.description; - const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channelName = scenario.description; + const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; - await channel.attach(); - // wait for getRoot() to resolve so the initial SYNC sequence is completed, - // as we're going to initiate a new one to test applying operations during SYNC sequence. - const root = await liveObjects.getRoot(); + await channel.attach(); + const root = await liveObjects.getRoot(); - await scenario.action({ root, liveObjectsHelper, channelName, channel }); - }, client); - }); + await scenario.action({ root, liveObjectsHelper, channelName, channel }); + }, client); + }, + ); const subscriptionCallbacksScenarios = [ { From a7df3b6f8f3a428863bcd235f3fa5ce75b8542c2 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 6 Dec 2024 03:06:57 +0000 Subject: [PATCH 081/166] Update `LiveMap.get` to have `undefined` as a possible return value This is a temporary solution for handling the access API for tombstoned objects. See comment in Confluence for the relevant discussion [1]. The return type for the LiveCounter.value is not changed as part of this commit, as the tombstoned LiveCounter has a value of 0 for its data. [1] https://ably.atlassian.net/wiki/spaces/LOB/pages/3556671496/LODR-026+Correctness+of+OBJECT_DELETE?focusedCommentId=3593928705 --- ably.d.ts | 6 ++++-- src/plugins/liveobjects/livemap.ts | 7 ++++++- .../browser/template/src/index-liveobjects.ts | 14 +++++++------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/ably.d.ts b/ably.d.ts index cd60057e74..eaccb3ccb6 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -2105,10 +2105,12 @@ export declare interface LiveMap extends LiveObject(key: TKey): T[TKey] extends StateValue ? T[TKey] : T[TKey] | undefined; + get(key: TKey): T[TKey] | undefined; /** * Returns the number of key/value pairs in the map. diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 7271beb5b5..b6899c2b7c 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -78,6 +78,7 @@ export class LiveMap extends LiveObject extends LiveObject(key: TKey): T[TKey] extends StateValue ? T[TKey] : T[TKey] | undefined { + get(key: TKey): T[TKey] | undefined { + if (this.isTombstoned()) { + return undefined as T[TKey]; + } + const element = this._dataRef.data.get(key); if (element === undefined) { diff --git a/test/package/browser/template/src/index-liveobjects.ts b/test/package/browser/template/src/index-liveobjects.ts index 00763d46be..4059cc01d8 100644 --- a/test/package/browser/template/src/index-liveobjects.ts +++ b/test/package/browser/template/src/index-liveobjects.ts @@ -24,14 +24,14 @@ globalThis.testAblyPackage = async function () { const size: number = root.size(); // check custom user provided typings via LiveObjectsTypes are working: + // any LiveMap.get() call can return undefined, as the LiveMap itself can be tombstoned (has empty state), + // or referenced object is tombstoned. // keys on a root: - const aNumber: number = root.get('numberKey'); - const aString: string = root.get('stringKey'); - const aBoolean: boolean = root.get('booleanKey'); - const couldBeUndefined: string | undefined = root.get('couldBeUndefined'); + const aNumber: number | undefined = root.get('numberKey'); + const aString: string | undefined = root.get('stringKey'); + const aBoolean: boolean | undefined = root.get('booleanKey'); + const userProvidedUndefined: string | undefined = root.get('couldBeUndefined'); // live objects on a root: - // LiveMap.get can still return undefined for LiveObject typed properties even if custom typings have them as non-optional. - // objects can be tombstoned and result in the undefined value const counter: Ably.LiveCounter | undefined = root.get('counterKey'); const map: LiveObjectsTypes['root']['mapKey'] | undefined = root.get('mapKey'); // check string literal types works @@ -63,5 +63,5 @@ globalThis.testAblyPackage = async function () { // check can provide custom types for the getRoot method, ignoring global LiveObjectsTypes interface const explicitRoot: Ably.LiveMap = await liveObjects.getRoot(); - const someOtherKey: string = explicitRoot.get('someOtherKey'); + const someOtherKey: string | undefined = explicitRoot.get('someOtherKey'); }; From f7609ec0f45c4dce5c5c9cba19a75c0ad4ad1365 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 11 Dec 2024 06:14:46 +0000 Subject: [PATCH 082/166] GC tombstoned map entries for LiveMap and objects in the global pool Resolves DTP-1024 --- src/plugins/liveobjects/defaults.ts | 10 ++ src/plugins/liveobjects/livecounter.ts | 8 + src/plugins/liveobjects/livemap.ts | 30 +++- src/plugins/liveobjects/liveobject.ts | 22 +++ src/plugins/liveobjects/liveobjects.ts | 4 + src/plugins/liveobjects/liveobjectspool.ts | 24 +++ test/common/modules/private_api_recorder.js | 7 + test/realtime/live_objects.test.js | 153 ++++++++++++++++++++ 8 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 src/plugins/liveobjects/defaults.ts diff --git a/src/plugins/liveobjects/defaults.ts b/src/plugins/liveobjects/defaults.ts new file mode 100644 index 0000000000..f5fb9d5c4f --- /dev/null +++ b/src/plugins/liveobjects/defaults.ts @@ -0,0 +1,10 @@ +export const DEFAULTS = { + gcInterval: 1000 * 60 * 5, // 5 minutes + /** + * Must be > 2 minutes to ensure we keep tombstones long enough to avoid the possibility of receiving an operation + * with an earlier origin timeserial that would not have been applied if the tombstone still existed. + * + * Applies both for map entries tombstones and object tombstones. + */ + gcGracePeriod: 1000 * 60 * 60 * 24, // 24 hours +}; diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index c96d59afe4..95b8dbe529 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -157,6 +157,14 @@ export class LiveCounter extends LiveObject return this._updateFromDataDiff(previousDataRef, this._dataRef); } + /** + * @internal + */ + onGCInterval(): void { + // nothing to GC for a counter object + return; + } + protected _getZeroValueData(): LiveCounterData { return { data: 0 }; } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index b6899c2b7c..378ef3b58f 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -1,6 +1,7 @@ import deepEqual from 'deep-equal'; import type * as API from '../../../ably'; +import { DEFAULTS } from './defaults'; import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; import { @@ -33,6 +34,10 @@ export type StateData = ObjectIdStateData | ValueStateData; export interface MapEntry { tombstone: boolean; + /** + * Can't use timeserial from the operation that deleted the entry for the same reason as for {@link LiveObject} tombstones, see explanation there. + */ + tombstonedAt: number | undefined; timeserial: string | undefined; data: StateData | undefined; } @@ -295,6 +300,22 @@ export class LiveMap extends LiveObject= DEFAULTS.gcGracePeriod) { + keysToDelete.push(key); + } + } + + keysToDelete.forEach((x) => this._dataRef.data.delete(x)); + } + protected _getZeroValueData(): LiveMapData { return { data: new Map() }; } @@ -459,11 +480,13 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject; protected _createOperationIsMerged: boolean; private _tombstone: boolean; + /** + * Even though the `timeserial` from the operation that deleted the object contains the timestamp value, + * the `timeserial` should be treated as an opaque string on the client, meaning we should not attempt to parse it. + * + * Therefore, we need to set our own timestamp using local clock when the object is deleted client-side. + * Strictly speaking, this does make an assumption about the client clock not being too heavily skewed behind the server, + * but it is an acceptable compromise for the time being, as the likelihood of encountering a race here is pretty low given the grace periods we use. + */ + private _tombstonedAt: number | undefined; protected constructor( protected _liveObjects: LiveObjects, @@ -108,6 +117,7 @@ export abstract class LiveObject< */ tombstone(): void { this._tombstone = true; + this._tombstonedAt = Date.now(); this._dataRef = this._getZeroValueData(); // TODO: emit "deleted" event so that end users get notified about this object getting deleted } @@ -119,6 +129,13 @@ export abstract class LiveObject< return this._tombstone; } + /** + * @internal + */ + tombstonedAt(): number | undefined { + return this._tombstonedAt; + } + /** * Returns true if the given origin timeserial indicates that the operation to which it belongs should be applied to the object. * @@ -168,6 +185,11 @@ export abstract class LiveObject< * @internal */ abstract overrideWithStateObject(stateObject: StateObject): TUpdate | LiveObjectUpdateNoop; + /** + * @internal + */ + abstract onGCInterval(): void; + protected abstract _getZeroValueData(): TData; /** * Calculate the update object based on the current Live Object data and incoming new data. diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 6ba3a0f2c7..06bfe21585 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -2,6 +2,7 @@ import type BaseClient from 'common/lib/client/baseclient'; import type RealtimeChannel from 'common/lib/client/realtimechannel'; import type EventEmitter from 'common/lib/util/eventemitter'; import type * as API from '../../../ably'; +import { DEFAULTS } from './defaults'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; @@ -26,6 +27,9 @@ export class LiveObjects { private _currentSyncCursor: string | undefined; private _bufferedStateOperations: StateMessage[]; + // Used by tests + static _DEFAULTS = DEFAULTS; + constructor(channel: RealtimeChannel) { this._channel = channel; this._client = channel.client; diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index eb42d47b4e..94f667fdb9 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -1,4 +1,5 @@ import type BaseClient from 'common/lib/client/baseclient'; +import { DEFAULTS } from './defaults'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; @@ -13,10 +14,16 @@ export const ROOT_OBJECT_ID = 'root'; export class LiveObjectsPool { private _client: BaseClient; private _pool: Map; + private _gcInterval: ReturnType; constructor(private _liveObjects: LiveObjects) { this._client = this._liveObjects.getClient(); this._pool = this._getInitialPool(); + this._gcInterval = setInterval(() => { + this._onGCInterval(); + }, DEFAULTS.gcInterval); + // call nodejs's Timeout.unref to not require Node.js event loop to remain active due to this interval. see https://nodejs.org/api/timers.html#timeoutunref + this._gcInterval.unref?.(); } get(objectId: string): LiveObject | undefined { @@ -68,4 +75,21 @@ export class LiveObjectsPool { pool.set(root.getObjectId(), root); return pool; } + + private _onGCInterval(): void { + const toDelete: string[] = []; + for (const [objectId, obj] of this._pool.entries()) { + // tombstoned objects should be removed from the pool if they have been tombstoned for longer than grace period. + // by removing them from the local pool, LiveObjects plugin no longer keeps a reference to those objects, allowing JS's + // Garbage Collection to eventually free the memory for those objects, provided the user no longer references them either. + if (obj.isTombstoned() && Date.now() - obj.tombstonedAt()! >= DEFAULTS.gcGracePeriod) { + toDelete.push(objectId); + continue; + } + + obj.onGCInterval(); + } + + toDelete.forEach((x) => this._pool.delete(x)); + } } diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index af478b713f..74158c9109 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -16,6 +16,9 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'call.Defaults.getPort', 'call.Defaults.normaliseOptions', 'call.EventEmitter.emit', + 'call.LiveObject.isTombstoned', + 'call.LiveObjects._liveObjectsPool._onGCInterval', + 'call.LiveObjects._liveObjectsPool.get', 'call.Message.decode', 'call.Message.encode', 'call.Platform.Config.push.storage.clear', @@ -72,6 +75,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'pass.clientOption.webSocketSlowTimeout', 'pass.clientOption.wsConnectivityCheckUrl', // actually ably-js public API (i.e. it’s in the TypeScript typings) but no other SDK has it. At the same time it's not entirely clear if websocket connectivity check should be considered an ably-js-specific functionality (as for other params above), so for the time being we consider it as private API 'read.Defaults.version', + 'read.LiveMap._dataRef.data', 'read.EventEmitter.events', 'read.Platform.Config.push', 'read.Realtime._transports', @@ -112,6 +116,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'read.transport.params.mode', 'read.transport.recvRequest.recvUri', 'read.transport.uri', + 'replace.LiveObjects._liveObjectsPool._onGCInterval', 'replace.channel.attachImpl', 'replace.channel.processMessage', 'replace.channel.sendMessage', @@ -128,6 +133,8 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'serialize.recoveryKey', 'write.Defaults.ENVIRONMENT', 'write.Defaults.wsConnectivityCheckUrl', + 'write.LiveObjects._DEFAULTS.gcGracePeriod', + 'write.LiveObjects._DEFAULTS.gcInterval', 'write.Platform.Config.push', // This implies using a mock implementation of the internal IPlatformPushConfig interface. Our mock (in push_channel_transport.js) then interacts with internal objects and private APIs of public objects to implement this interface; I haven’t added annotations for that private API usage, since there wasn’t an easy way to pass test context information into the mock. I think that for now we can just say that if we wanted to get rid of this private API usage, then we’d need to remove this mock entirely. 'write.auth.authOptions.requestHeaders', 'write.auth.key', diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 22164ef795..d1b3551ff8 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -12,6 +12,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const createPM = Ably.makeProtocolMessageFromDeserialized({ LiveObjectsPlugin }); const liveObjectsFixturesChannel = 'liveobjects_fixtures'; const nextTick = Ably.Realtime.Platform.Config.nextTick; + const gcIntervalOriginal = LiveObjectsPlugin.LiveObjects._DEFAULTS.gcInterval; + const gcGracePeriodOriginal = LiveObjectsPlugin.LiveObjects._DEFAULTS.gcGracePeriod; function RealtimeWithLiveObjects(helper, options) { return helper.AblyRealtime({ ...options, plugins: { LiveObjects: LiveObjectsPlugin } }); @@ -2585,6 +2587,157 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); }, client); }); + + const tombstonesGCScenarios = [ + // for the next tests we need to access the private API of LiveObjects plugin in order to verify that tombstoned entities were indeed deleted after the GC grace period. + // public API hides that kind of information from the user and returns undefined for tombstoned entities even if realtime client still keeps a reference to them. + { + description: 'tombstoned object is removed from the pool after the GC grace period', + action: async (ctx) => { + const { liveObjectsHelper, channelName, channel, liveObjects, helper, waitForGCCycles } = ctx; + + // send a CREATE op, this add an object to the pool + const { objectId } = await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.counterCreateOp({ count: 1 }), + ); + + expect(liveObjects._liveObjectsPool.get(objectId), 'Check object exists in the pool after creation').to + .exist; + + // inject OBJECT_DELETE for the object. this should tombstone the object and make it inaccessible to the end user, but still keep it in memory in the local pool + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [liveObjectsHelper.objectDeleteOp({ objectId })], + }); + + helper.recordPrivateApi('call.LiveObjects._liveObjectsPool.get'); + expect( + liveObjects._liveObjectsPool.get(objectId), + 'Check object exists in the pool immediately after OBJECT_DELETE', + ).to.exist; + helper.recordPrivateApi('call.LiveObjects._liveObjectsPool.get'); + helper.recordPrivateApi('call.LiveObject.isTombstoned'); + expect(liveObjects._liveObjectsPool.get(objectId).isTombstoned()).to.equal( + true, + `Check object's "tombstone" flag is set to "true" after OBJECT_DELETE`, + ); + + // we expect 2 cycles to guarantee that grace period has expired, which will always be true based on the test config used + await waitForGCCycles(2); + + // object should be removed from the local pool entirely now, as the GC grace period has passed + helper.recordPrivateApi('call.LiveObjects._liveObjectsPool.get'); + expect( + liveObjects._liveObjectsPool.get(objectId), + 'Check object exists does not exist in the pool after the GC grace period expiration', + ).to.not.exist; + }, + }, + + { + description: 'tombstoned map entry is removed from the LiveMap after the GC grace period', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, helper, waitForGCCycles } = ctx; + + // set a key on a root + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { value: 'bar' } }), + ); + + expect(root.get('foo')).to.equal('bar', 'Check key "foo" exists on root after MAP_SET'); + + // remove the key from the root. this should tombstone the map entry and make it inaccessible to the end user, but still keep it in memory in the underlying map + await liveObjectsHelper.stateRequest( + channelName, + liveObjectsHelper.mapRemoveOp({ objectId: 'root', key: 'foo' }), + ); + + expect(root.get('foo'), 'Check key "foo" is inaccessible via public API on root after MAP_REMOVE').to.not + .exist; + helper.recordPrivateApi('read.LiveMap._dataRef.data'); + expect( + root._dataRef.data.get('foo'), + 'Check map entry for "foo" exists on root in the underlying data immediately after MAP_REMOVE', + ).to.exist; + helper.recordPrivateApi('read.LiveMap._dataRef.data'); + expect( + root._dataRef.data.get('foo').tombstone, + 'Check map entry for "foo" on root has "tombstone" flag set to "true" after MAP_REMOVE', + ).to.exist; + + // we expect 2 cycles to guarantee that grace period has expired, which will always be true based on the test config used + await waitForGCCycles(2); + + // the entry should be removed from the underlying map now + helper.recordPrivateApi('read.LiveMap._dataRef.data'); + expect( + root._dataRef.data.get('foo'), + 'Check map entry for "foo" does not exist on root in the underlying data after the GC grace period expiration', + ).to.not.exist; + }, + }, + ]; + + /** @nospec */ + forScenarios(tombstonesGCScenarios, async function (helper, scenario) { + try { + helper.recordPrivateApi('write.LiveObjects._DEFAULTS.gcInterval'); + LiveObjectsPlugin.LiveObjects._DEFAULTS.gcInterval = 500; + helper.recordPrivateApi('write.LiveObjects._DEFAULTS.gcGracePeriod'); + LiveObjectsPlugin.LiveObjects._DEFAULTS.gcGracePeriod = 250; + + const liveObjectsHelper = new LiveObjectsHelper(helper); + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channelName = scenario.description; + const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + // helper function to spy on the GC interval callback and wait for a specific number of GC cycles. + // returns a promise which will resolve when required number of cycles have happened. + const waitForGCCycles = (cycles) => { + const onGCIntervalOriginal = liveObjects._liveObjectsPool._onGCInterval; + let gcCalledTimes = 0; + return new Promise((resolve) => { + helper.recordPrivateApi('replace.LiveObjects._liveObjectsPool._onGCInterval'); + liveObjects._liveObjectsPool._onGCInterval = function () { + helper.recordPrivateApi('call.LiveObjects._liveObjectsPool._onGCInterval'); + onGCIntervalOriginal.call(this); + + gcCalledTimes++; + if (gcCalledTimes >= cycles) { + resolve(); + liveObjects._liveObjectsPool._onGCInterval = onGCIntervalOriginal; + } + }; + }); + }; + + await scenario.action({ + root, + liveObjectsHelper, + channelName, + channel, + liveObjects, + helper, + waitForGCCycles, + }); + }, client); + } finally { + helper.recordPrivateApi('write.LiveObjects._DEFAULTS.gcInterval'); + LiveObjectsPlugin.LiveObjects._DEFAULTS.gcInterval = gcIntervalOriginal; + helper.recordPrivateApi('write.LiveObjects._DEFAULTS.gcGracePeriod'); + LiveObjectsPlugin.LiveObjects._DEFAULTS.gcGracePeriod = gcGracePeriodOriginal; + } + }); }); /** @nospec */ From f4073b9b0c8dd19dcd24ee0131690e462f0f7e6f Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 12 Dec 2024 01:42:51 +0000 Subject: [PATCH 083/166] Refactor Message and PresenceMessage payload encoding This reduces code duplication and prepares for the changes in next commits where a StateMessage from LiveObjects plugin will be using these encoding functions to encode its state. --- src/common/lib/types/message.ts | 170 +++++++++++++++++------- src/common/lib/types/presencemessage.ts | 40 +++--- 2 files changed, 142 insertions(+), 68 deletions(-) diff --git a/src/common/lib/types/message.ts b/src/common/lib/types/message.ts index a1a449942d..e8bad29f84 100644 --- a/src/common/lib/types/message.ts +++ b/src/common/lib/types/message.ts @@ -131,41 +131,120 @@ export async function fromEncodedArray( ); } -async function encrypt(msg: T, options: CipherOptions): Promise { - let data = msg.data, - encoding = msg.encoding, - cipher = options.channelCipher; - - encoding = encoding ? encoding + '/' : ''; - if (!Platform.BufferUtils.isBuffer(data)) { - data = Platform.BufferUtils.utf8Encode(String(data)); - encoding = encoding + 'utf-8/'; - } - const ciphertext = await cipher.encrypt(data); - msg.data = ciphertext; - msg.encoding = encoding + 'cipher+' + cipher.algorithm; +async function encrypt(msg: T, cipherOptions: CipherOptions): Promise { + const { data, encoding } = await encryptData(msg.data, msg.encoding, cipherOptions); + msg.data = data; + msg.encoding = encoding; return msg; } -export async function encode(msg: T, options: CipherOptions): Promise { - const data = msg.data; +export async function encryptData( + data: any, + encoding: string | null | undefined, + cipherOptions: CipherOptions, +): Promise<{ data: any; encoding: string | null | undefined }> { + let cipher = cipherOptions.channelCipher; + let dataToEncrypt = data; + let finalEncoding = encoding ? encoding + '/' : ''; + + if (!Platform.BufferUtils.isBuffer(dataToEncrypt)) { + dataToEncrypt = Platform.BufferUtils.utf8Encode(String(dataToEncrypt)); + finalEncoding = finalEncoding + 'utf-8/'; + } + + const ciphertext = await cipher.encrypt(dataToEncrypt); + finalEncoding = finalEncoding + 'cipher+' + cipher.algorithm; + + return { + data: ciphertext, + encoding: finalEncoding, + }; +} + +/** + * Protocol agnostic encoding and encryption of the message's payload. Mutates the message. + * Implements RSL4 (only parts that are common for all protocols), and RSL5. + * + * Since this encoding function is protocol agnostic, it won't apply the final encodings + * required by the protocol used by the client (like encoding binary data to the appropriate representation). + */ +export async function encode(msg: T, cipherOptions: CipherOptions): Promise { + const { data, encoding } = encodeData(msg.data, msg.encoding); + msg.data = data; + msg.encoding = encoding; + + if (cipherOptions != null && cipherOptions.cipher) { + return encrypt(msg, cipherOptions); + } else { + return msg; + } +} + +/** + * Protocol agnostic encoding of the provided payload data. Implements RSL4 (only parts that are common for all protocols). + */ +export function encodeData( + data: any, + encoding: string | null | undefined, +): { data: any; encoding: string | null | undefined } { + // RSL4a, supported types const nativeDataType = typeof data == 'string' || Platform.BufferUtils.isBuffer(data) || data === null || data === undefined; - if (!nativeDataType) { - if (Utils.isObject(data) || Array.isArray(data)) { - msg.data = JSON.stringify(data); - msg.encoding = msg.encoding ? msg.encoding + '/json' : 'json'; - } else { - throw new ErrorInfo('Data type is unsupported', 40013, 400); - } + if (nativeDataType) { + // nothing to do with the native data types at this point + return { + data, + encoding, + }; } - if (options != null && options.cipher) { - return encrypt(msg, options); - } else { - return msg; + if (Utils.isObject(data) || Array.isArray(data)) { + // RSL4c3 and RSL4d3, encode objects and arrays as strings + return { + data: JSON.stringify(data), + encoding: encoding ? encoding + '/json' : 'json', + }; + } + + // RSL4a, throw an error for unsupported types + throw new ErrorInfo('Data type is unsupported', 40013, 400); +} + +/** + * Prepares the payload data to be transmitted over the wire to Ably. + * Encodes the data depending on the selected protocol format. + * + * Implements RSL4c1 and RSL4d1 + */ +export function encodeDataForWireProtocol( + data: any, + encoding: string | null | undefined, + format: Utils.Format, +): { data: any; encoding: string | null | undefined } { + if (!data || !Platform.BufferUtils.isBuffer(data)) { + // no transformation required for non-buffer payloads + return { + data, + encoding, + }; + } + + if (format === Utils.Format.msgpack) { + // RSL4c1 + // BufferUtils.toBuffer returns a datatype understandable by that platform's msgpack implementation: + // Buffer in node, Uint8Array in browsers + return { + data: Platform.BufferUtils.toBuffer(data), + encoding, + }; } + + // RSL4d1, encode binary payload as base64 string + return { + data: Platform.BufferUtils.base64Encode(data), + encoding: encoding ? encoding + '/base64' : 'base64', + }; } export async function encodeArray(messages: Array, options: CipherOptions): Promise> { @@ -178,6 +257,8 @@ export async function decode( message: Message | PresenceMessage, inputContext: CipherOptions | EncodingDecodingContext | ChannelOptions, ): Promise { + // data can be decoded partially and throw an error on a later decoding step. + // so we need to reassign the data and encoding values we got, and only then throw an error if there is one const { data, encoding, error } = await decodeData(message.data, message.encoding, inputContext); message.data = data; message.encoding = encoding; @@ -187,6 +268,9 @@ export async function decode( } } +/** + * Implements RSL6 + */ export async function decodeData( data: any, encoding: string | null | undefined, @@ -199,8 +283,8 @@ export async function decodeData( const context = normaliseContext(inputContext); let lastPayload = data; let decodedData = data; - let finalEncoding: string | null | undefined = encoding; - let decodingError: ErrorInfo | undefined = undefined; + let finalEncoding = encoding; + let decodingError: ErrorInfo | undefined; if (encoding) { const xforms = encoding.split('/'); @@ -378,27 +462,19 @@ class Message { operation?: API.Operation; /** - * Overload toJSON() to intercept JSON.stringify() - * @return {*} + * Overload toJSON() to intercept JSON.stringify(). + * + * This will prepare the message to be transmitted over the wire to Ably. + * It will encode the data payload according to the wire protocol used on the client. + * It will transform any client-side enum string representations into their corresponding numbers, if needed (like "action" fields). */ toJSON() { - /* encode data to base64 if present and we're returning real JSON; - * although msgpack calls toJSON(), we know it is a stringify() - * call if it has a non-empty arguments list */ - let encoding = this.encoding; - let data = this.data; - if (data && Platform.BufferUtils.isBuffer(data)) { - if (arguments.length > 0) { - /* stringify call */ - encoding = encoding ? encoding + '/base64' : 'base64'; - data = Platform.BufferUtils.base64Encode(data); - } else { - /* Called by msgpack. toBuffer returns a datatype understandable by - * that platform's msgpack implementation (Buffer in node, Uint8Array - * in browsers) */ - data = Platform.BufferUtils.toBuffer(data); - } - } + // we can infer the format used by client by inspecting with what arguments this method was called. + // if JSON protocol is being used, the JSON.stringify() will be called and this toJSON() method will have a non-empty arguments list. + // MSGPack protocol implementation also calls toJSON(), but with an empty arguments list. + const format = arguments.length > 0 ? Utils.Format.json : Utils.Format.msgpack; + const { data, encoding } = encodeDataForWireProtocol(this.data, this.encoding, format); + return { name: this.name, id: this.id, diff --git a/src/common/lib/types/presencemessage.ts b/src/common/lib/types/presencemessage.ts index 34e0d2d06d..144ceb3d2f 100644 --- a/src/common/lib/types/presencemessage.ts +++ b/src/common/lib/types/presencemessage.ts @@ -1,6 +1,12 @@ import Logger from '../util/logger'; import Platform from 'common/platform'; -import { encode as encodeMessage, decode as decodeMessage, getMessagesSize, CipherOptions } from './message'; +import { + encode as encodeMessage, + decode as decodeMessage, + getMessagesSize, + CipherOptions, + encodeDataForWireProtocol, +} from './message'; import * as Utils from '../util/utils'; import * as API from '../../../../ably'; import { MsgPack } from 'common/types/msgpack'; @@ -128,8 +134,11 @@ class PresenceMessage { } /** - * Overload toJSON() to intercept JSON.stringify() - * @return {*} + * Overload toJSON() to intercept JSON.stringify(). + * + * This will prepare the message to be transmitted over the wire to Ably. + * It will encode the data payload according to the wire protocol used on the client. + * It will transform any client-side enum string representations into their corresponding numbers, if needed (like "action" fields). */ toJSON(): { id?: string; @@ -139,30 +148,19 @@ class PresenceMessage { encoding?: string; extras?: any; } { - /* encode data to base64 if present and we're returning real JSON; - * although msgpack calls toJSON(), we know it is a stringify() - * call if it has a non-empty arguments list */ - let data = this.data as string | Buffer | Uint8Array; - let encoding = this.encoding; - if (data && Platform.BufferUtils.isBuffer(data)) { - if (arguments.length > 0) { - /* stringify call */ - encoding = encoding ? encoding + '/base64' : 'base64'; - data = Platform.BufferUtils.base64Encode(data); - } else { - /* Called by msgpack. toBuffer returns a datatype understandable by - * that platform's msgpack implementation (Buffer in node, Uint8Array - * in browsers) */ - data = Platform.BufferUtils.toBuffer(data); - } - } + // we can infer the format used by client by inspecting with what arguments this method was called. + // if JSON protocol is being used, the JSON.stringify() will be called and this toJSON() method will have a non-empty arguments list. + // MSGPack protocol implementation also calls toJSON(), but with an empty arguments list. + const format = arguments.length > 0 ? Utils.Format.json : Utils.Format.msgpack; + const { data, encoding } = encodeDataForWireProtocol(this.data, this.encoding, format); + return { id: this.id, clientId: this.clientId, /* Convert presence action back to an int for sending to Ably */ action: toActionValue(this.action as string), data: data, - encoding: encoding, + encoding: encoding!, extras: this.extras, }; } From a1bf1ef2fe87aecb8f9dbeedb1937c0b0d3d5b19 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 14 Jan 2025 20:25:55 +0000 Subject: [PATCH 084/166] Expose message encoding functions from Message and on a BaseClient --- src/common/lib/client/baseclient.ts | 2 ++ src/common/lib/client/realtimechannel.ts | 4 ++-- src/common/lib/types/message.ts | 7 +++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/common/lib/client/baseclient.ts b/src/common/lib/client/baseclient.ts index be2cde2056..c79677b654 100644 --- a/src/common/lib/client/baseclient.ts +++ b/src/common/lib/client/baseclient.ts @@ -18,6 +18,7 @@ import { HTTPRequestImplementations } from 'platform/web/lib/http/http'; import { FilteredSubscriptions } from './filteredsubscriptions'; import type { LocalDevice } from 'plugins/push/pushactivation'; import EventEmitter from '../util/eventemitter'; +import { MessageEncoding } from '../types/message'; type BatchResult = API.BatchResult; type BatchPublishSpec = API.BatchPublishSpec; @@ -181,6 +182,7 @@ class BaseClient { Defaults = Defaults; Utils = Utils; EventEmitter = EventEmitter; + MessageEncoding = MessageEncoding; } export default BaseClient; diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index 427b6398e6..8a599dbaac 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -12,10 +12,10 @@ import Message, { fromValuesArray as messagesFromValuesArray, encodeArray as encodeMessagesArray, decode as decodeMessage, - decodeData, getMessagesSize, CipherOptions, EncodingDecodingContext, + MessageEncoding, } from '../types/message'; import ChannelStateChange from './channelstatechange'; import ErrorInfo, { PartialErrorInfo } from '../types/errorinfo'; @@ -615,7 +615,7 @@ class RealtimeChannel extends EventEmitter { const options = this.channelOptions; await this._decodeAndPrepareMessages(message, stateMessages, (msg) => this.client._LiveObjectsPlugin - ? this.client._LiveObjectsPlugin.StateMessage.decode(msg, options, decodeData) + ? this.client._LiveObjectsPlugin.StateMessage.decode(msg, options, MessageEncoding.decodeData) : Utils.throwMissingPluginError('LiveObjects'), ); diff --git a/src/common/lib/types/message.ts b/src/common/lib/types/message.ts index e8bad29f84..3abad6b2dd 100644 --- a/src/common/lib/types/message.ts +++ b/src/common/lib/types/message.ts @@ -442,6 +442,13 @@ export function getMessagesSize(messages: Message[]): number { return total; } +export const MessageEncoding = { + encryptData, + encodeData, + encodeDataForWireProtocol, + decodeData, +}; + class Message { name?: string; id?: string; From 2b48dffb0682f4de54e7ebd5f2d26a1551f6cb99 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 14 Jan 2025 20:42:49 +0000 Subject: [PATCH 085/166] Refactor `StateMessage` to use message encoding functions exported via `MessageEncoding` --- src/common/lib/client/realtimechannel.ts | 2 +- src/common/lib/types/protocolmessage.ts | 12 +- src/plugins/liveobjects/statemessage.ts | 170 ++++++++++------------- 3 files changed, 83 insertions(+), 101 deletions(-) diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index 8a599dbaac..e7d5afe445 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -615,7 +615,7 @@ class RealtimeChannel extends EventEmitter { const options = this.channelOptions; await this._decodeAndPrepareMessages(message, stateMessages, (msg) => this.client._LiveObjectsPlugin - ? this.client._LiveObjectsPlugin.StateMessage.decode(msg, options, MessageEncoding.decodeData) + ? this.client._LiveObjectsPlugin.StateMessage.decode(msg, options, MessageEncoding) : Utils.throwMissingPluginError('LiveObjects'), ); diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index b0617cdf85..fb473177aa 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -3,13 +3,16 @@ import * as API from '../../../../ably'; import { PresenceMessagePlugin } from '../client/modularplugins'; import * as Utils from '../util/utils'; import ErrorInfo from './errorinfo'; -import Message, { fromValues as messageFromValues, fromValuesArray as messagesFromValuesArray } from './message'; +import Message, { + fromValues as messageFromValues, + fromValuesArray as messagesFromValuesArray, + MessageEncoding, +} from './message'; import PresenceMessage, { fromValues as presenceMessageFromValues, fromValuesArray as presenceMessagesFromValuesArray, } from './presencemessage'; import type * as LiveObjectsPlugin from 'plugins/liveobjects'; -import Platform from '../../platform'; export const actions = { HEARTBEAT: 0, @@ -128,7 +131,7 @@ export function fromDeserialized( state = deserialized.state as LiveObjectsPlugin.StateMessage[]; if (state) { for (let i = 0; i < state.length; i++) { - state[i] = liveObjectsPlugin.StateMessage.fromValues(state[i], Platform); + state[i] = liveObjectsPlugin.StateMessage.fromValues(state[i], Utils, MessageEncoding); } } } @@ -177,7 +180,8 @@ export function stringify( if (msg.presence && presenceMessagePlugin) result += '; presence=' + toStringArray(presenceMessagePlugin.presenceMessagesFromValuesArray(msg.presence)); if (msg.state && liveObjectsPlugin) { - result += '; state=' + toStringArray(liveObjectsPlugin.StateMessage.fromValuesArray(msg.state, Platform)); + result += + '; state=' + toStringArray(liveObjectsPlugin.StateMessage.fromValuesArray(msg.state, Utils, MessageEncoding)); } if (msg.error) result += '; error=' + ErrorInfo.fromValues(msg.error).toString(); if (msg.auth && msg.auth.accessToken) result += '; token=' + msg.auth.accessToken; diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 5dd409811d..46d783ade3 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -1,7 +1,12 @@ -import type { decodeData } from 'common/lib/types/message'; -import type Platform from 'common/platform'; +import type { MessageEncoding } from 'common/lib/types/message'; +import type * as Utils from 'common/lib/util/utils'; import type { ChannelOptions } from 'common/types/channel'; +export type StateDataEncodeFunction = ( + value: StateValue | undefined, + encoding: string | undefined, +) => { value: StateValue | undefined; encoding: string | undefined }; + export enum StateOperationAction { MAP_CREATE = 0, MAP_SET = 1, @@ -146,46 +151,60 @@ export class StateMessage { /** Site code corresponding to this message's timeserial */ siteCode?: string; - constructor(private _platform: typeof Platform) {} + constructor( + private _utils: typeof Utils, + private _messageEncoding: typeof MessageEncoding, + ) {} + /** + * Mutates the provided StateMessage and decodes all data entries in the message + */ static async decode( message: StateMessage, inputContext: ChannelOptions, - decodeDataFn: typeof decodeData, + messageEncoding: typeof MessageEncoding, ): Promise { // TODO: decide how to handle individual errors from decoding values. currently we throw first ever error we get if (message.object?.map?.entries) { - await StateMessage._decodeMapEntries(message.object.map.entries, inputContext, decodeDataFn); + await StateMessage._decodeMapEntries(message.object.map.entries, inputContext, messageEncoding); } if (message.object?.createOp?.map?.entries) { - await StateMessage._decodeMapEntries(message.object.createOp.map.entries, inputContext, decodeDataFn); + await StateMessage._decodeMapEntries(message.object.createOp.map.entries, inputContext, messageEncoding); } if (message.object?.createOp?.mapOp?.data && 'value' in message.object.createOp.mapOp.data) { - await StateMessage._decodeStateData(message.object.createOp.mapOp.data, inputContext, decodeDataFn); + await StateMessage._decodeStateData(message.object.createOp.mapOp.data, inputContext, messageEncoding); } if (message.operation?.map?.entries) { - await StateMessage._decodeMapEntries(message.operation.map.entries, inputContext, decodeDataFn); + await StateMessage._decodeMapEntries(message.operation.map.entries, inputContext, messageEncoding); } if (message.operation?.mapOp?.data && 'value' in message.operation.mapOp.data) { - await StateMessage._decodeStateData(message.operation.mapOp.data, inputContext, decodeDataFn); + await StateMessage._decodeStateData(message.operation.mapOp.data, inputContext, messageEncoding); } } - static fromValues(values: StateMessage | Record, platform: typeof Platform): StateMessage { - return Object.assign(new StateMessage(platform), values); + static fromValues( + values: StateMessage | Record, + utils: typeof Utils, + messageEncoding: typeof MessageEncoding, + ): StateMessage { + return Object.assign(new StateMessage(utils, messageEncoding), values); } - static fromValuesArray(values: unknown[], platform: typeof Platform): StateMessage[] { + static fromValuesArray( + values: (StateMessage | Record)[], + utils: typeof Utils, + messageEncoding: typeof MessageEncoding, + ): StateMessage[] { const count = values.length; const result = new Array(count); for (let i = 0; i < count; i++) { - result[i] = StateMessage.fromValues(values[i] as Record, platform); + result[i] = StateMessage.fromValues(values[i], utils, messageEncoding); } return result; @@ -194,19 +213,23 @@ export class StateMessage { private static async _decodeMapEntries( mapEntries: Record, inputContext: ChannelOptions, - decodeDataFn: typeof decodeData, + messageEncoding: typeof MessageEncoding, ): Promise { for (const entry of Object.values(mapEntries)) { - await StateMessage._decodeStateData(entry.data, inputContext, decodeDataFn); + await StateMessage._decodeStateData(entry.data, inputContext, messageEncoding); } } private static async _decodeStateData( stateData: StateData, inputContext: ChannelOptions, - decodeDataFn: typeof decodeData, + messageEncoding: typeof MessageEncoding, ): Promise { - const { data, encoding, error } = await decodeDataFn(stateData.value, stateData.encoding, inputContext); + const { data, encoding, error } = await messageEncoding.decodeData( + stateData.value, + stateData.encoding, + inputContext, + ); stateData.value = data; stateData.encoding = encoding ?? undefined; @@ -216,9 +239,8 @@ export class StateMessage { } private static _encodeStateOperation( - platform: typeof Platform, stateOperation: StateOperation, - withBase64Encoding: boolean, + encodeFn: StateDataEncodeFunction, ): StateOperation { // deep copy "stateOperation" object so we can modify the copy here. // buffer values won't be correctly copied, so we will need to set them again explictly. @@ -226,32 +248,20 @@ export class StateMessage { if (stateOperationCopy.mapOp?.data && 'value' in stateOperationCopy.mapOp.data) { // use original "stateOperation" object when encoding values, so we have access to the original buffer values. - stateOperationCopy.mapOp.data = StateMessage._encodeStateData( - platform, - stateOperation.mapOp?.data!, - withBase64Encoding, - ); + stateOperationCopy.mapOp.data = StateMessage._encodeStateData(stateOperation.mapOp?.data!, encodeFn); } if (stateOperationCopy.map?.entries) { Object.entries(stateOperationCopy.map.entries).forEach(([key, entry]) => { // use original "stateOperation" object when encoding values, so we have access to original buffer values. - entry.data = StateMessage._encodeStateData( - platform, - stateOperation?.map?.entries?.[key].data!, - withBase64Encoding, - ); + entry.data = StateMessage._encodeStateData(stateOperation?.map?.entries?.[key].data!, encodeFn); }); } return stateOperationCopy; } - private static _encodeStateObject( - platform: typeof Platform, - stateObject: StateObject, - withBase64Encoding: boolean, - ): StateObject { + private static _encodeStateObject(stateObject: StateObject, encodeFn: StateDataEncodeFunction): StateObject { // deep copy "stateObject" object so we can modify the copy here. // buffer values won't be correctly copied, so we will need to set them again explictly. const stateObjectCopy = JSON.parse(JSON.stringify(stateObject)) as StateObject; @@ -259,71 +269,34 @@ export class StateMessage { if (stateObjectCopy.map?.entries) { Object.entries(stateObjectCopy.map.entries).forEach(([key, entry]) => { // use original "stateObject" object when encoding values, so we have access to original buffer values. - entry.data = StateMessage._encodeStateData( - platform, - stateObject?.map?.entries?.[key].data!, - withBase64Encoding, - ); + entry.data = StateMessage._encodeStateData(stateObject?.map?.entries?.[key].data!, encodeFn); }); } if (stateObjectCopy.createOp) { // use original "stateObject" object when encoding values, so we have access to original buffer values. - stateObjectCopy.createOp = StateMessage._encodeStateOperation( - platform, - stateObject.createOp!, - withBase64Encoding, - ); + stateObjectCopy.createOp = StateMessage._encodeStateOperation(stateObject.createOp!, encodeFn); } return stateObjectCopy; } - private static _encodeStateData(platform: typeof Platform, data: StateData, withBase64Encoding: boolean): StateData { - const { value, encoding } = StateMessage._encodeStateValue( - platform, - data?.value, - data?.encoding, - withBase64Encoding, - ); - return { - ...data, - value, - encoding, - }; - } + private static _encodeStateData(data: StateData, encodeFn: StateDataEncodeFunction): StateData { + const { value: newValue, encoding: newEncoding } = encodeFn(data?.value, data?.encoding); - private static _encodeStateValue( - platform: typeof Platform, - value: StateValue | undefined, - encoding: string | undefined, - withBase64Encoding: boolean, - ): { - value: StateValue | undefined; - encoding: string | undefined; - } { - if (!value || !platform.BufferUtils.isBuffer(value)) { - return { value, encoding }; - } - - if (withBase64Encoding) { - return { - value: platform.BufferUtils.base64Encode(value), - encoding: encoding ? encoding + '/base64' : 'base64', - }; - } - - // toBuffer returns a datatype understandable by - // that platform's msgpack implementation (Buffer in node, Uint8Array in browsers) return { - value: platform.BufferUtils.toBuffer(value), - encoding, + ...data, + value: newValue, + encoding: newEncoding!, }; } /** - * Overload toJSON() to intercept JSON.stringify() - * @return {*} + * Overload toJSON() to intercept JSON.stringify(). + * + * This will prepare the message to be transmitted over the wire to Ably. + * It will encode the data payload according to the wire protocol used on the client. + * It will transform any client-side enum string representations into their corresponding numbers, if needed (like "action" fields). */ toJSON(): { id?: string; @@ -332,19 +305,24 @@ export class StateMessage { object?: StateObject; extras?: any; } { - // need to encode buffer data to base64 if present and if we're returning a real JSON. - // although msgpack also calls toJSON() directly, - // we know it is a JSON.stringify() call if we have a non-empty arguments list. - // if withBase64Encoding = true - JSON.stringify() call - // if withBase64Encoding = false - we were called by msgpack - const withBase64Encoding = arguments.length > 0; - - const encodedOperation = this.operation - ? StateMessage._encodeStateOperation(this._platform, this.operation, withBase64Encoding) - : undefined; - const encodedObject = this.object - ? StateMessage._encodeStateObject(this._platform, this.object, withBase64Encoding) - : undefined; + // we can infer the format used by client by inspecting with what arguments this method was called. + // if JSON protocol is being used, the JSON.stringify() will be called and this toJSON() method will have a non-empty arguments list. + // MSGPack protocol implementation also calls toJSON(), but with an empty arguments list. + const format = arguments.length > 0 ? this._utils.Format.json : this._utils.Format.msgpack; + const encodeFn: StateDataEncodeFunction = (value, encoding) => { + const { data: newValue, encoding: newEncoding } = this._messageEncoding.encodeDataForWireProtocol( + value, + encoding, + format, + ); + return { + value: newValue, + encoding: newEncoding!, + }; + }; + + const encodedOperation = this.operation ? StateMessage._encodeStateOperation(this.operation, encodeFn) : undefined; + const encodedObject = this.object ? StateMessage._encodeStateObject(this.object, encodeFn) : undefined; return { id: this.id, From 195e67a7486c11786a59790cce78662d6d7973d7 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 14 Jan 2025 20:43:57 +0000 Subject: [PATCH 086/166] Add `StateMessage.encode` method --- src/plugins/liveobjects/statemessage.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 46d783ade3..429256ee37 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -156,6 +156,28 @@ export class StateMessage { private _messageEncoding: typeof MessageEncoding, ) {} + /** + * Protocol agnostic encoding of the state message's data entries. + * Mutates the provided StateMessage. + * + * Uses encoding functions from regular `Message` processing. + */ + static async encode(message: StateMessage, messageEncoding: typeof MessageEncoding): Promise { + const encodeFn: StateDataEncodeFunction = (value, encoding) => { + const { data: newValue, encoding: newEncoding } = messageEncoding.encodeData(value, encoding); + + return { + value: newValue, + encoding: newEncoding!, + }; + }; + + message.operation = message.operation ? StateMessage._encodeStateOperation(message.operation, encodeFn) : undefined; + message.object = message.object ? StateMessage._encodeStateObject(message.object, encodeFn) : undefined; + + return message; + } + /** * Mutates the provided StateMessage and decodes all data entries in the message */ From 5b365bca4df0f8cdb00c295225444aa762001692 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 14 Jan 2025 21:37:34 +0000 Subject: [PATCH 087/166] Add `RealtimeChannel.sendState` and mark STATE messages as those that require an ACK --- src/common/lib/client/realtimechannel.ts | 11 +++++++++++ src/common/lib/transport/protocol.ts | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index e7d5afe445..1208453aec 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -511,6 +511,17 @@ class RealtimeChannel extends EventEmitter { this.sendMessage(msg, callback); } + sendState(state: StateMessage[]): Promise { + return new Promise((resolve, reject) => { + const msg = protocolMessageFromValues({ + action: actions.STATE, + channel: this.name, + state, + }); + this.sendMessage(msg, (err) => (err ? reject(err) : resolve())); + }); + } + // Access to this method is synchronised by ConnectionManager#processChannelMessage, in order to synchronise access to the state stored in _decodingContext. async processMessage(message: ProtocolMessage): Promise { if ( diff --git a/src/common/lib/transport/protocol.ts b/src/common/lib/transport/protocol.ts index 88a5947e4a..47a71171ff 100644 --- a/src/common/lib/transport/protocol.ts +++ b/src/common/lib/transport/protocol.ts @@ -20,7 +20,8 @@ export class PendingMessage { this.merged = false; const action = message.action; this.sendAttempted = false; - this.ackRequired = action == actions.MESSAGE || action == actions.PRESENCE; + this.ackRequired = + typeof action === 'number' && [actions.MESSAGE, actions.PRESENCE, actions.STATE].includes(action); } } From f2804e6a6a7e96082948c4b2168ce7cfb2176ba4 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 14 Jan 2025 21:48:01 +0000 Subject: [PATCH 088/166] Add object-level write API for `LiveMap` and `LiveCounter` to update existing objects --- src/plugins/liveobjects/livecounter.ts | 54 ++++++++++++- src/plugins/liveobjects/livemap.ts | 100 ++++++++++++++++++++++++- src/plugins/liveobjects/liveobjects.ts | 17 +++++ 3 files changed, 165 insertions(+), 6 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 95b8dbe529..10542f0677 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -36,6 +36,56 @@ export class LiveCounter extends LiveObject return this._dataRef.data; } + /** + * Send a COUNTER_INC operation to the realtime system to increment a value on this LiveCounter object. + * + * This does not modify the underlying data of this LiveCounter object. Instead, the change will be applied when + * the published COUNTER_INC operation is echoed back to the client and applied to the object following the regular + * operation application procedure. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. + */ + async increment(amount: number): Promise { + const stateMessage = this.createCounterIncMessage(amount); + return this._liveObjects.publish([stateMessage]); + } + + /** + * @internal + */ + createCounterIncMessage(amount: number): StateMessage { + if (typeof amount !== 'number' || !isFinite(amount)) { + throw new this._client.ErrorInfo('Counter value increment should be a valid number', 40013, 400); + } + + const stateMessage = StateMessage.fromValues( + { + operation: { + action: StateOperationAction.COUNTER_INC, + objectId: this.getObjectId(), + counterOp: { amount }, + }, + }, + this._client.Utils, + this._client.MessageEncoding, + ); + + return stateMessage; + } + + /** + * Alias for calling {@link LiveCounter.increment | LiveCounter.increment(-amount)} + */ + async decrement(amount: number): Promise { + // do an explicit type safety check here before negating the amount value, + // so we don't unintentionally change the type sent by a user + if (typeof amount !== 'number' || !isFinite(amount)) { + throw new this._client.ErrorInfo('Counter value decrement should be a valid number', 40013, 400); + } + + return this.increment(-amount); + } + /** * @internal */ @@ -55,7 +105,7 @@ export class LiveCounter extends LiveObject this._client.logger, this._client.Logger.LOG_MICRO, 'LiveCounter.applyOperation()', - `skipping ${op.action} op: op timeserial ${opOriginTimeserial.toString()} <= site timeserial ${this._siteTimeserials[opSiteCode]?.toString()}; objectId=${this._objectId}`, + `skipping ${op.action} op: op timeserial ${opOriginTimeserial.toString()} <= site timeserial ${this._siteTimeserials[opSiteCode]?.toString()}; objectId=${this.getObjectId()}`, ); return; } @@ -202,7 +252,7 @@ export class LiveCounter extends LiveObject this._client.logger, this._client.Logger.LOG_MICRO, 'LiveCounter._applyCounterCreate()', - `skipping applying COUNTER_CREATE op on a counter instance as it was already applied before; objectId=${this._objectId}`, + `skipping applying COUNTER_CREATE op on a counter instance as it was already applied before; objectId=${this.getObjectId()}`, ); return { noop: true }; } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 378ef3b58f..79b291d3d6 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -153,6 +153,98 @@ export class LiveMap extends LiveObject(key: TKey, value: T[TKey]): Promise { + const stateMessage = this.createMapSetMessage(key, value); + return this._liveObjects.publish([stateMessage]); + } + + /** + * @internal + */ + createMapSetMessage(key: TKey, value: T[TKey]): StateMessage { + if (typeof key !== 'string') { + throw new this._client.ErrorInfo('Map key should be string', 40013, 400); + } + + if ( + typeof value !== 'string' && + typeof value !== 'number' && + typeof value !== 'boolean' && + !this._client.Platform.BufferUtils.isBuffer(value) && + !(value instanceof LiveObject) + ) { + throw new this._client.ErrorInfo('Map value data type is unsupported', 40013, 400); + } + + const stateData: StateData = + value instanceof LiveObject + ? ({ objectId: value.getObjectId() } as ObjectIdStateData) + : ({ value } as ValueStateData); + + const stateMessage = StateMessage.fromValues( + { + operation: { + action: StateOperationAction.MAP_SET, + objectId: this.getObjectId(), + mapOp: { + key, + data: stateData, + }, + }, + }, + this._client.Utils, + this._client.MessageEncoding, + ); + + return stateMessage; + } + + /** + * Send a MAP_REMOVE operation to the realtime system to tombstone a key on this LiveMap object. + * + * This does not modify the underlying data of this LiveMap object. Instead, the change will be applied when + * the published MAP_REMOVE operation is echoed back to the client and applied to the object following the regular + * operation application procedure. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. + */ + async remove(key: TKey): Promise { + const stateMessage = this.createMapRemoveMessage(key); + return this._liveObjects.publish([stateMessage]); + } + + /** + * @internal + */ + createMapRemoveMessage(key: TKey): StateMessage { + if (typeof key !== 'string') { + throw new this._client.ErrorInfo('Map key should be string', 40013, 400); + } + + const stateMessage = StateMessage.fromValues( + { + operation: { + action: StateOperationAction.MAP_REMOVE, + objectId: this.getObjectId(), + mapOp: { key }, + }, + }, + this._client.Utils, + this._client.MessageEncoding, + ); + + return stateMessage; + } + /** * @internal */ @@ -172,7 +264,7 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject extends LiveObject { + if (!this._channel.connectionManager.activeState()) { + throw this._channel.connectionManager.getError(); + } + + if (this._channel.state === 'failed' || this._channel.state === 'suspended') { + throw this._client.ErrorInfo.fromValues(this._channel.invalidStateError()); + } + + stateMessages.forEach((x) => StateMessage.encode(x, this._client.MessageEncoding)); + + return this._channel.sendState(stateMessages); + } + private _startNewSync(syncId?: string, syncCursor?: string): void { // need to discard all buffered state operation messages on new sync start this._bufferedStateOperations = []; From f8563cbb3ea0158e7648f83d72fb8799240d6124 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 15 Jan 2025 01:10:45 +0000 Subject: [PATCH 089/166] Add tests for object-level write API --- test/realtime/live_objects.test.js | 376 ++++++++++++++++++++++++++++- 1 file changed, 374 insertions(+), 2 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index d1b3551ff8..0d60a6c194 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -57,6 +57,17 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], return `${paddedTimestamp}-${paddedCounter}@${seriesId}` + (paddedIndex ? `:${paddedIndex}` : ''); } + async function expectRejectedWith(fn, errorStr) { + let verifiedError = false; + try { + await fn(); + } catch (error) { + expect(error.message).to.have.string(errorStr); + verifiedError = true; + } + expect(verifiedError, 'Expected async function to throw an error').to.be.true; + } + describe('realtime/live_objects', function () { this.timeout(60 * 1000); @@ -518,7 +529,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { name: 'negativeMaxSafeIntegerCounter', count: -Number.MAX_SAFE_INTEGER }, ]; - const stateSyncSequenceScanarios = [ + const stateSyncSequenceScenarios = [ { description: 'STATE_SYNC sequence with state object "tombstone" property creates tombstoned object', action: async (ctx) => { @@ -2060,9 +2071,370 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, ]; + const writeApiScenarios = [ + { + description: 'LiveCounter.increment sends COUNTER_INC operation', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: liveObjectsHelper.counterCreateOp(), + }); + + const counter = root.get('counter'); + const increments = [ + 1, // value=1 + 10, // value=11 + -11, // value=0 + -1, // value=-1 + -10, // value=-11 + 11, // value=0 + Number.MAX_SAFE_INTEGER, // value=9007199254740991 + -Number.MAX_SAFE_INTEGER, // value=0 + -Number.MAX_SAFE_INTEGER, // value=-9007199254740991 + ]; + let expectedCounterValue = 0; + + for (let i = 0; i < increments.length; i++) { + const increment = increments[i]; + expectedCounterValue += increment; + await counter.increment(increment); + + expect(counter.value()).to.equal( + expectedCounterValue, + `Check counter has correct value after ${i + 1} LiveCounter.increment calls`, + ); + } + }, + }, + + { + description: 'LiveCounter.increment throws on invalid input', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: liveObjectsHelper.counterCreateOp(), + }); + + const counter = root.get('counter'); + + await expectRejectedWith( + async () => counter.increment(), + 'Counter value increment should be a valid number', + ); + await expectRejectedWith( + async () => counter.increment(null), + 'Counter value increment should be a valid number', + ); + await expectRejectedWith( + async () => counter.increment(Number.NaN), + 'Counter value increment should be a valid number', + ); + await expectRejectedWith( + async () => counter.increment(Number.POSITIVE_INFINITY), + 'Counter value increment should be a valid number', + ); + await expectRejectedWith( + async () => counter.increment(Number.NEGATIVE_INFINITY), + 'Counter value increment should be a valid number', + ); + await expectRejectedWith( + async () => counter.increment('foo'), + 'Counter value increment should be a valid number', + ); + await expectRejectedWith( + async () => counter.increment(BigInt(1)), + 'Counter value increment should be a valid number', + ); + await expectRejectedWith( + async () => counter.increment(true), + 'Counter value increment should be a valid number', + ); + await expectRejectedWith( + async () => counter.increment(Symbol()), + 'Counter value increment should be a valid number', + ); + await expectRejectedWith( + async () => counter.increment({}), + 'Counter value increment should be a valid number', + ); + await expectRejectedWith( + async () => counter.increment([]), + 'Counter value increment should be a valid number', + ); + await expectRejectedWith( + async () => counter.increment(counter), + 'Counter value increment should be a valid number', + ); + }, + }, + + { + description: 'LiveCounter.decrement sends COUNTER_INC operation', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: liveObjectsHelper.counterCreateOp(), + }); + + const counter = root.get('counter'); + const decrements = [ + 1, // value=-1 + 10, // value=-11 + -11, // value=0 + -1, // value=1 + -10, // value=11 + 11, // value=0 + Number.MAX_SAFE_INTEGER, // value=-9007199254740991 + -Number.MAX_SAFE_INTEGER, // value=0 + -Number.MAX_SAFE_INTEGER, // value=9007199254740991 + ]; + let expectedCounterValue = 0; + + for (let i = 0; i < decrements.length; i++) { + const decrement = decrements[i]; + expectedCounterValue -= decrement; + await counter.decrement(decrement); + + expect(counter.value()).to.equal( + expectedCounterValue, + `Check counter has correct value after ${i + 1} LiveCounter.decrement calls`, + ); + } + }, + }, + + { + description: 'LiveCounter.decrement throws on invalid input', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: liveObjectsHelper.counterCreateOp(), + }); + + const counter = root.get('counter'); + + await expectRejectedWith( + async () => counter.decrement(), + 'Counter value decrement should be a valid number', + ); + await expectRejectedWith( + async () => counter.decrement(null), + 'Counter value decrement should be a valid number', + ); + await expectRejectedWith( + async () => counter.decrement(Number.NaN), + 'Counter value decrement should be a valid number', + ); + await expectRejectedWith( + async () => counter.decrement(Number.POSITIVE_INFINITY), + 'Counter value decrement should be a valid number', + ); + await expectRejectedWith( + async () => counter.decrement(Number.NEGATIVE_INFINITY), + 'Counter value decrement should be a valid number', + ); + await expectRejectedWith( + async () => counter.decrement('foo'), + 'Counter value decrement should be a valid number', + ); + await expectRejectedWith( + async () => counter.decrement(BigInt(1)), + 'Counter value decrement should be a valid number', + ); + await expectRejectedWith( + async () => counter.decrement(true), + 'Counter value decrement should be a valid number', + ); + await expectRejectedWith( + async () => counter.decrement(Symbol()), + 'Counter value decrement should be a valid number', + ); + await expectRejectedWith( + async () => counter.decrement({}), + 'Counter value decrement should be a valid number', + ); + await expectRejectedWith( + async () => counter.decrement([]), + 'Counter value decrement should be a valid number', + ); + await expectRejectedWith( + async () => counter.decrement(counter), + 'Counter value decrement should be a valid number', + ); + }, + }, + + { + description: 'LiveMap.set sends MAP_SET operation with primitive values', + action: async (ctx) => { + const { root } = ctx; + + await Promise.all( + primitiveKeyData.map(async (keyData) => { + const value = keyData.data.encoding ? BufferUtils.base64Decode(keyData.data.value) : keyData.data.value; + await root.set(keyData.key, value); + }), + ); + + // check everything is applied correctly + primitiveKeyData.forEach((keyData) => { + if (keyData.data.encoding) { + expect( + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), + `Check root has correct value for "${keyData.key}" key after LiveMap.set call`, + ).to.be.true; + } else { + expect(root.get(keyData.key)).to.equal( + keyData.data.value, + `Check root has correct value for "${keyData.key}" key after LiveMap.set call`, + ); + } + }); + }, + }, + + { + description: 'LiveMap.set sends MAP_SET operation with reference to another LiveObject', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: liveObjectsHelper.counterCreateOp(), + }); + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: liveObjectsHelper.mapCreateOp(), + }); + + const counter = root.get('counter'); + const map = root.get('map'); + + await root.set('counter2', counter); + await root.set('map2', map); + + expect(root.get('counter2')).to.equal( + counter, + 'Check can set a reference to a LiveCounter object on a root via a LiveMap.set call', + ); + expect(root.get('map2')).to.equal( + map, + 'Check can set a reference to a LiveMap object on a root via a LiveMap.set call', + ); + }, + }, + + { + description: 'LiveMap.set throws on invalid input', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: liveObjectsHelper.mapCreateOp(), + }); + + const map = root.get('map'); + + await expectRejectedWith(async () => map.set(), 'Map key should be string'); + await expectRejectedWith(async () => map.set(null), 'Map key should be string'); + await expectRejectedWith(async () => map.set(1), 'Map key should be string'); + await expectRejectedWith(async () => map.set(BigInt(1)), 'Map key should be string'); + await expectRejectedWith(async () => map.set(true), 'Map key should be string'); + await expectRejectedWith(async () => map.set(Symbol()), 'Map key should be string'); + await expectRejectedWith(async () => map.set({}), 'Map key should be string'); + await expectRejectedWith(async () => map.set([]), 'Map key should be string'); + await expectRejectedWith(async () => map.set(map), 'Map key should be string'); + + await expectRejectedWith(async () => map.set('key'), 'Map value data type is unsupported'); + await expectRejectedWith(async () => map.set('key', null), 'Map value data type is unsupported'); + await expectRejectedWith(async () => map.set('key', BigInt(1)), 'Map value data type is unsupported'); + await expectRejectedWith(async () => map.set('key', Symbol()), 'Map value data type is unsupported'); + await expectRejectedWith(async () => map.set('key', {}), 'Map value data type is unsupported'); + await expectRejectedWith(async () => map.set('key', []), 'Map value data type is unsupported'); + }, + }, + + { + description: 'LiveMap.remove sends MAP_REMOVE operation', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: liveObjectsHelper.mapCreateOp({ + entries: { + foo: { data: { value: 1 } }, + bar: { data: { value: 1 } }, + baz: { data: { value: 1 } }, + }, + }), + }); + + const map = root.get('map'); + + await map.remove('foo'); + await map.remove('bar'); + + expect(map.get('foo'), 'Check can remove a key from a root via a LiveMap.remove call').to.not.exist; + expect(map.get('bar'), 'Check can remove a key from a root via a LiveMap.remove call').to.not.exist; + expect( + map.get('baz'), + 'Check non-removed keys are still present on a root after LiveMap.remove call for another keys', + ).to.equal(1); + }, + }, + + { + description: 'LiveMap.remove throws on invalid input', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName } = ctx; + + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: liveObjectsHelper.mapCreateOp(), + }); + + const map = root.get('map'); + + await expectRejectedWith(async () => map.remove(), 'Map key should be string'); + await expectRejectedWith(async () => map.remove(null), 'Map key should be string'); + await expectRejectedWith(async () => map.remove(1), 'Map key should be string'); + await expectRejectedWith(async () => map.remove(BigInt(1)), 'Map key should be string'); + await expectRejectedWith(async () => map.remove(true), 'Map key should be string'); + await expectRejectedWith(async () => map.remove(Symbol()), 'Map key should be string'); + await expectRejectedWith(async () => map.remove({}), 'Map key should be string'); + await expectRejectedWith(async () => map.remove([]), 'Map key should be string'); + await expectRejectedWith(async () => map.remove(map), 'Map key should be string'); + }, + }, + ]; + /** @nospec */ forScenarios( - [...stateSyncSequenceScanarios, ...applyOperationsScenarios, ...applyOperationsDuringSyncScenarios], + [ + ...stateSyncSequenceScenarios, + ...applyOperationsScenarios, + ...applyOperationsDuringSyncScenarios, + ...writeApiScenarios, + ], async function (helper, scenario) { const liveObjectsHelper = new LiveObjectsHelper(helper); const client = RealtimeWithLiveObjects(helper); From 78ded6c832a28e7a071449addc772a3428362313 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 00:52:28 +0000 Subject: [PATCH 090/166] Spelling fix --- src/plugins/liveobjects/livecounter.ts | 2 +- src/plugins/liveobjects/livemap.ts | 4 ++-- src/plugins/liveobjects/statemessage.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 10542f0677..8432035830 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -185,7 +185,7 @@ export class LiveCounter extends LiveObject this._siteTimeserials = stateObject.siteTimeserials ?? {}; if (this.isTombstoned()) { - // this object is tombstoned. this is a terminal state which can't be overriden. skip the rest of state object message processing + // this object is tombstoned. this is a terminal state which can't be overridden. skip the rest of state object message processing return { noop: true }; } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 79b291d3d6..e12c9a1c4a 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -370,7 +370,7 @@ export class LiveMap extends LiveObject extends LiveObject Date: Wed, 15 Jan 2025 23:21:37 +0000 Subject: [PATCH 091/166] Add `IBufferUtils.sha256` method --- src/common/types/IBufferUtils.ts | 1 + src/platform/nodejs/lib/util/bufferutils.ts | 11 +++++++---- src/platform/web/lib/util/bufferutils.ts | 7 ++++++- src/platform/web/lib/util/hmac-sha256.ts | 4 ++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/common/types/IBufferUtils.ts b/src/common/types/IBufferUtils.ts index be573a44f8..0ba48892e0 100644 --- a/src/common/types/IBufferUtils.ts +++ b/src/common/types/IBufferUtils.ts @@ -19,5 +19,6 @@ export default interface IBufferUtils { * Returns ArrayBuffer on browser and Buffer on Node.js */ arrayBufferViewToBuffer: (arrayBufferView: ArrayBufferView) => Bufferlike; + sha256(message: Bufferlike): Output; hmacSha256(message: Bufferlike, key: Bufferlike): Output; } diff --git a/src/platform/nodejs/lib/util/bufferutils.ts b/src/platform/nodejs/lib/util/bufferutils.ts index 52b46ba2b1..86baa4632d 100644 --- a/src/platform/nodejs/lib/util/bufferutils.ts +++ b/src/platform/nodejs/lib/util/bufferutils.ts @@ -70,14 +70,17 @@ class BufferUtils implements IBufferUtils { return Buffer.from(string, 'utf8'); } + sha256(message: Bufferlike): Output { + const messageBuffer = this.toBuffer(message); + + return crypto.createHash('SHA256').update(messageBuffer).digest(); + } + hmacSha256(message: Bufferlike, key: Bufferlike): Output { const messageBuffer = this.toBuffer(message); const keyBuffer = this.toBuffer(key); - const hmac = crypto.createHmac('SHA256', keyBuffer); - hmac.update(messageBuffer); - - return hmac.digest(); + return crypto.createHmac('SHA256', keyBuffer).update(messageBuffer).digest(); } } diff --git a/src/platform/web/lib/util/bufferutils.ts b/src/platform/web/lib/util/bufferutils.ts index 062e663515..bc42a273e4 100644 --- a/src/platform/web/lib/util/bufferutils.ts +++ b/src/platform/web/lib/util/bufferutils.ts @@ -1,6 +1,6 @@ import Platform from 'common/platform'; import IBufferUtils from 'common/types/IBufferUtils'; -import { hmac as hmacSha256 } from './hmac-sha256'; +import { hmac as hmacSha256, sha256 } from './hmac-sha256'; /* Most BufferUtils methods that return a binary object return an ArrayBuffer * The exception is toBuffer, which returns a Uint8Array */ @@ -195,6 +195,11 @@ class BufferUtils implements IBufferUtils { return this.toArrayBuffer(arrayBufferView); } + sha256(message: Bufferlike): Output { + const hash = sha256(this.toBuffer(message)); + return this.toArrayBuffer(hash); + } + hmacSha256(message: Bufferlike, key: Bufferlike): Output { const hash = hmacSha256(this.toBuffer(key), this.toBuffer(message)); return this.toArrayBuffer(hash); diff --git a/src/platform/web/lib/util/hmac-sha256.ts b/src/platform/web/lib/util/hmac-sha256.ts index dd2ac76711..ac69b6ee1e 100644 --- a/src/platform/web/lib/util/hmac-sha256.ts +++ b/src/platform/web/lib/util/hmac-sha256.ts @@ -102,7 +102,7 @@ function rightRotate(word: number, bits: number) { return (word >>> bits) | (word << (32 - bits)); } -function sha256(data: Uint8Array) { +export function sha256(data: Uint8Array): Uint8Array { // Copy default state var STATE = DEFAULT_STATE.slice(); @@ -185,7 +185,7 @@ function sha256(data: Uint8Array) { ); } -export function hmac(key: Uint8Array, data: Uint8Array) { +export function hmac(key: Uint8Array, data: Uint8Array): Uint8Array { if (key.length > 64) key = sha256(key); if (key.length < 64) { From 3707c6786b5dfb5b24f5cd9c539b575c769d214f Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 15 Jan 2025 23:32:20 +0000 Subject: [PATCH 092/166] Add `IBufferUtils.base64UrlEncode` method --- src/common/types/IBufferUtils.ts | 1 + src/platform/nodejs/lib/util/bufferutils.ts | 4 ++++ src/platform/web/lib/util/bufferutils.ts | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/src/common/types/IBufferUtils.ts b/src/common/types/IBufferUtils.ts index 0ba48892e0..8cd1d39aa3 100644 --- a/src/common/types/IBufferUtils.ts +++ b/src/common/types/IBufferUtils.ts @@ -8,6 +8,7 @@ export default interface IBufferUtils { toBuffer: (buffer: Bufferlike) => ToBufferOutput; toArrayBuffer: (buffer: Bufferlike) => ArrayBuffer; base64Encode: (buffer: Bufferlike) => string; + base64UrlEncode: (buffer: Bufferlike) => string; base64Decode: (string: string) => Output; hexEncode: (buffer: Bufferlike) => string; hexDecode: (string: string) => Output; diff --git a/src/platform/nodejs/lib/util/bufferutils.ts b/src/platform/nodejs/lib/util/bufferutils.ts index 86baa4632d..8c93f4ef34 100644 --- a/src/platform/nodejs/lib/util/bufferutils.ts +++ b/src/platform/nodejs/lib/util/bufferutils.ts @@ -17,6 +17,10 @@ class BufferUtils implements IBufferUtils { return this.toBuffer(buffer).toString('base64'); } + base64UrlEncode(buffer: Bufferlike): string { + return this.toBuffer(buffer).toString('base64url'); + } + areBuffersEqual(buffer1: Bufferlike, buffer2: Bufferlike): boolean { if (!buffer1 || !buffer2) return false; return this.toBuffer(buffer1).compare(this.toBuffer(buffer2)) == 0; diff --git a/src/platform/web/lib/util/bufferutils.ts b/src/platform/web/lib/util/bufferutils.ts index bc42a273e4..1d7af7d694 100644 --- a/src/platform/web/lib/util/bufferutils.ts +++ b/src/platform/web/lib/util/bufferutils.ts @@ -116,6 +116,11 @@ class BufferUtils implements IBufferUtils { return this.uint8ViewToBase64(this.toBuffer(buffer)); } + base64UrlEncode(buffer: Bufferlike): string { + // base64url encoding is based on regular base64 with following changes: https://base64.guru/standards/base64url + return this.base64Encode(buffer).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); + } + base64Decode(str: string): Output { if (ArrayBuffer && Platform.Config.atob) { return this.base64ToArrayBuffer(str); From ea3fd7ac7931e04ad1950d27ec7b5501466ef13e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 15 Jan 2025 23:17:55 +0000 Subject: [PATCH 093/166] Add `ObjectId.msTimestamp` property --- src/plugins/liveobjects/objectid.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/plugins/liveobjects/objectid.ts b/src/plugins/liveobjects/objectid.ts index c968eefd60..e92db1dbab 100644 --- a/src/plugins/liveobjects/objectid.ts +++ b/src/plugins/liveobjects/objectid.ts @@ -11,6 +11,7 @@ export class ObjectId { private constructor( readonly type: LiveObjectType, readonly hash: string, + readonly msTimestamp: number, ) {} /** @@ -21,8 +22,8 @@ export class ObjectId { throw new client.ErrorInfo('Invalid object id string', 50000, 500); } - const [type, hash] = objectId.split(':'); - if (!type || !hash) { + const [type, rest] = objectId.split(':'); + if (!type || !rest) { throw new client.ErrorInfo('Invalid object id string', 50000, 500); } @@ -30,6 +31,19 @@ export class ObjectId { throw new client.ErrorInfo(`Invalid object type in object id: ${objectId}`, 50000, 500); } - return new ObjectId(type as LiveObjectType, hash); + const [hash, msTimestamp] = rest.split('@'); + if (!hash || !msTimestamp) { + throw new client.ErrorInfo('Invalid object id string', 50000, 500); + } + + if (!Number.isInteger(Number.parseInt(msTimestamp))) { + throw new client.ErrorInfo('Invalid object id string', 50000, 500); + } + + return new ObjectId(type as LiveObjectType, hash, Number.parseInt(msTimestamp)); + } + + toString(): string { + return `${this.type}:${this.hash}@${this.msTimestamp}`; } } From 21f1c26d0d453956c6ecad1f0aa0a4a90ca53cf2 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 15 Jan 2025 23:33:57 +0000 Subject: [PATCH 094/166] Add `ObjectId.fromInitialValue` method --- src/plugins/liveobjects/objectid.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/plugins/liveobjects/objectid.ts b/src/plugins/liveobjects/objectid.ts index e92db1dbab..eb10b3f1fc 100644 --- a/src/plugins/liveobjects/objectid.ts +++ b/src/plugins/liveobjects/objectid.ts @@ -1,4 +1,5 @@ import type BaseClient from 'common/lib/client/baseclient'; +import type Platform from 'common/platform'; export type LiveObjectType = 'map' | 'counter'; @@ -14,6 +15,21 @@ export class ObjectId { readonly msTimestamp: number, ) {} + static fromInitialValue( + platform: typeof Platform, + objectType: LiveObjectType, + encodedInitialValue: string, + nonce: string, + msTimestamp: number, + ): ObjectId { + const valueForHashBuffer = platform.BufferUtils.utf8Encode(`${encodedInitialValue}:${nonce}`); + const hashBuffer = platform.BufferUtils.sha256(valueForHashBuffer); + + const hash = platform.BufferUtils.base64UrlEncode(hashBuffer); + + return new ObjectId(objectType, hash, msTimestamp); + } + /** * Create ObjectId instance from hashed object id string. */ From c8784a6482e69df5993a7c72adee916df2a94bd2 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 05:11:48 +0000 Subject: [PATCH 095/166] Fix incorrect buffer type for `StateValue` --- src/plugins/liveobjects/statemessage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 1a1ca9fb16..573bba72be 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -21,7 +21,7 @@ export enum MapSemantics { } /** A StateValue represents a concrete leaf value in a state object graph. */ -export type StateValue = string | number | boolean | Buffer | Uint8Array; +export type StateValue = string | number | boolean | Buffer | ArrayBuffer; /** StateData captures a value in a state object. */ export interface StateData { From 6f5b171e8e15c9c276b41b02739cac82b30652ac Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 00:52:47 +0000 Subject: [PATCH 096/166] Add `initialValue` and `initialValueEncoding` to `StateMessage` --- src/plugins/liveobjects/statemessage.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 573bba72be..4bf40c8e91 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -104,6 +104,15 @@ export interface StateOperation { * that has been hashed with the type and initial value to create the object ID. */ nonce?: string; + /** + * The initial value bytes for the object. These bytes should be used along with the nonce + * and timestamp to create the object ID. Frontdoor will use this to verify the object ID. + * After verification the bytes will be decoded into the Map or Counter objects and + * the initialValue, nonce, and initialValueEncoding will be removed. + */ + initialValue?: Buffer | ArrayBuffer; + /** The initial value encoding defines how the initialValue should be interpreted. */ + initialValueEncoding?: string; } /** A StateObject describes the instantaneous state of an object. */ From 7fdbe7379e0f758016f6cd27c9c10b9e0c7d2e3d Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 00:53:05 +0000 Subject: [PATCH 097/166] Add `StateMessage.encodeInitialValue` method --- src/plugins/liveobjects/statemessage.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 4bf40c8e91..f2564d597d 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -241,6 +241,19 @@ export class StateMessage { return result; } + static encodeInitialValue( + utils: typeof Utils, + initialValue: Partial, + ): { + encodedInitialValue: string; + format: Utils.Format; + } { + return { + encodedInitialValue: JSON.stringify(initialValue), + format: utils.Format.json, + }; + } + private static async _decodeMapEntries( mapEntries: Record, inputContext: ChannelOptions, From d9d8c385eeed3e68f89f0f9f014c976733fa5ee6 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 01:00:50 +0000 Subject: [PATCH 098/166] Update `LiveObjectsPool.createZeroValueObjectIfNotExists` to also return created object --- src/plugins/liveobjects/liveobjectspool.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index 94f667fdb9..428bdf867f 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -48,9 +48,10 @@ export class LiveObjectsPool { this._pool = this._getInitialPool(); } - createZeroValueObjectIfNotExists(objectId: string): void { - if (this.get(objectId)) { - return; + createZeroValueObjectIfNotExists(objectId: string): LiveObject { + const existingObject = this.get(objectId); + if (existingObject) { + return existingObject; } const parsedObjectId = ObjectId.fromString(this._client, objectId); @@ -67,6 +68,7 @@ export class LiveObjectsPool { } this.set(objectId, zeroValueObject); + return zeroValueObject; } private _getInitialPool(): Map { From ee0ff57d0d1e3f7d45a444d32abec604f664dcc6 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 01:03:36 +0000 Subject: [PATCH 099/166] Refactor `StateMessage` creation in Live Objects --- src/plugins/liveobjects/livecounter.ts | 48 ++++---- src/plugins/liveobjects/livemap.ts | 158 ++++++++++++++----------- 2 files changed, 117 insertions(+), 89 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 8432035830..a681f6087f 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -32,47 +32,49 @@ export class LiveCounter extends LiveObject return obj; } - value(): number { - return this._dataRef.data; - } - - /** - * Send a COUNTER_INC operation to the realtime system to increment a value on this LiveCounter object. - * - * This does not modify the underlying data of this LiveCounter object. Instead, the change will be applied when - * the published COUNTER_INC operation is echoed back to the client and applied to the object following the regular - * operation application procedure. - * - * @returns A promise which resolves upon receiving the ACK message for the published operation message. - */ - async increment(amount: number): Promise { - const stateMessage = this.createCounterIncMessage(amount); - return this._liveObjects.publish([stateMessage]); - } - /** * @internal */ - createCounterIncMessage(amount: number): StateMessage { + static createCounterIncMessage(liveObjects: LiveObjects, objectId: string, amount: number): StateMessage { + const client = liveObjects.getClient(); + if (typeof amount !== 'number' || !isFinite(amount)) { - throw new this._client.ErrorInfo('Counter value increment should be a valid number', 40013, 400); + throw new client.ErrorInfo('Counter value increment should be a valid number', 40013, 400); } const stateMessage = StateMessage.fromValues( { operation: { action: StateOperationAction.COUNTER_INC, - objectId: this.getObjectId(), + objectId, counterOp: { amount }, }, }, - this._client.Utils, - this._client.MessageEncoding, + client.Utils, + client.MessageEncoding, ); return stateMessage; } + value(): number { + return this._dataRef.data; + } + + /** + * Send a COUNTER_INC operation to the realtime system to increment a value on this LiveCounter object. + * + * This does not modify the underlying data of this LiveCounter object. Instead, the change will be applied when + * the published COUNTER_INC operation is echoed back to the client and applied to the object following the regular + * operation application procedure. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. + */ + async increment(amount: number): Promise { + const stateMessage = LiveCounter.createCounterIncMessage(this._liveObjects, this.getObjectId(), amount); + return this._liveObjects.publish([stateMessage]); + } + /** * Alias for calling {@link LiveCounter.increment | LiveCounter.increment(-amount)} */ diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index e12c9a1c4a..3a6efe90f5 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -80,6 +80,96 @@ export class LiveMap extends LiveObject( + liveObjects: LiveObjects, + objectId: string, + key: TKey, + value: API.LiveMapType[TKey], + ): StateMessage { + const client = liveObjects.getClient(); + + LiveMap.validateKeyValue(liveObjects, key, value); + + const stateData: StateData = + value instanceof LiveObject + ? ({ objectId: value.getObjectId() } as ObjectIdStateData) + : ({ value } as ValueStateData); + + const stateMessage = StateMessage.fromValues( + { + operation: { + action: StateOperationAction.MAP_SET, + objectId, + mapOp: { + key, + data: stateData, + }, + }, + }, + client.Utils, + client.MessageEncoding, + ); + + return stateMessage; + } + + /** + * @internal + */ + static createMapRemoveMessage( + liveObjects: LiveObjects, + objectId: string, + key: TKey, + ): StateMessage { + const client = liveObjects.getClient(); + + if (typeof key !== 'string') { + throw new client.ErrorInfo('Map key should be string', 40013, 400); + } + + const stateMessage = StateMessage.fromValues( + { + operation: { + action: StateOperationAction.MAP_REMOVE, + objectId, + mapOp: { key }, + }, + }, + client.Utils, + client.MessageEncoding, + ); + + return stateMessage; + } + + /** + * @internal + */ + static validateKeyValue( + liveObjects: LiveObjects, + key: TKey, + value: API.LiveMapType[TKey], + ): void { + const client = liveObjects.getClient(); + + if (typeof key !== 'string') { + throw new client.ErrorInfo('Map key should be string', 40013, 400); + } + + if ( + typeof value !== 'string' && + typeof value !== 'number' && + typeof value !== 'boolean' && + !client.Platform.BufferUtils.isBuffer(value) && + !(value instanceof LiveObject) + ) { + throw new client.ErrorInfo('Map value data type is unsupported', 40013, 400); + } + } + /** * Returns the value associated with the specified key in the underlying Map object. * @@ -163,51 +253,10 @@ export class LiveMap extends LiveObject(key: TKey, value: T[TKey]): Promise { - const stateMessage = this.createMapSetMessage(key, value); + const stateMessage = LiveMap.createMapSetMessage(this._liveObjects, this.getObjectId(), key, value); return this._liveObjects.publish([stateMessage]); } - /** - * @internal - */ - createMapSetMessage(key: TKey, value: T[TKey]): StateMessage { - if (typeof key !== 'string') { - throw new this._client.ErrorInfo('Map key should be string', 40013, 400); - } - - if ( - typeof value !== 'string' && - typeof value !== 'number' && - typeof value !== 'boolean' && - !this._client.Platform.BufferUtils.isBuffer(value) && - !(value instanceof LiveObject) - ) { - throw new this._client.ErrorInfo('Map value data type is unsupported', 40013, 400); - } - - const stateData: StateData = - value instanceof LiveObject - ? ({ objectId: value.getObjectId() } as ObjectIdStateData) - : ({ value } as ValueStateData); - - const stateMessage = StateMessage.fromValues( - { - operation: { - action: StateOperationAction.MAP_SET, - objectId: this.getObjectId(), - mapOp: { - key, - data: stateData, - }, - }, - }, - this._client.Utils, - this._client.MessageEncoding, - ); - - return stateMessage; - } - /** * Send a MAP_REMOVE operation to the realtime system to tombstone a key on this LiveMap object. * @@ -218,33 +267,10 @@ export class LiveMap extends LiveObject(key: TKey): Promise { - const stateMessage = this.createMapRemoveMessage(key); + const stateMessage = LiveMap.createMapRemoveMessage(this._liveObjects, this.getObjectId(), key); return this._liveObjects.publish([stateMessage]); } - /** - * @internal - */ - createMapRemoveMessage(key: TKey): StateMessage { - if (typeof key !== 'string') { - throw new this._client.ErrorInfo('Map key should be string', 40013, 400); - } - - const stateMessage = StateMessage.fromValues( - { - operation: { - action: StateOperationAction.MAP_REMOVE, - objectId: this.getObjectId(), - mapOp: { key }, - }, - }, - this._client.Utils, - this._client.MessageEncoding, - ); - - return stateMessage; - } - /** * @internal */ From 5d5874dc6cd1a94a0a922c062447873f80d394fc Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 01:28:14 +0000 Subject: [PATCH 100/166] Move `getTimestamp` and related methods from `Auth` to `BaseClient` --- src/common/lib/client/auth.ts | 33 ++++++++--------------------- src/common/lib/client/baseclient.ts | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/common/lib/client/auth.ts b/src/common/lib/client/auth.ts index 67e7704f2b..d457ce10c3 100644 --- a/src/common/lib/client/auth.ts +++ b/src/common/lib/client/auth.ts @@ -775,7 +775,7 @@ class Auth { capability = tokenParams.capability || ''; if (!request.timestamp) { - request.timestamp = await this.getTimestamp(authOptions && authOptions.queryTime); + request.timestamp = await this._getTimestamp(authOptions && authOptions.queryTime); } /* nonce */ @@ -832,28 +832,6 @@ class Auth { } } - /** - * Get the current time based on the local clock, - * or if the option queryTime is true, return the server time. - * The server time offset from the local time is stored so that - * only one request to the server to get the time is ever needed - */ - async getTimestamp(queryTime: boolean): Promise { - if (!this.isTimeOffsetSet() && (queryTime || this.authOptions.queryTime)) { - return this.client.time(); - } else { - return this.getTimestampUsingOffset(); - } - } - - getTimestampUsingOffset() { - return Date.now() + (this.client.serverTimeOffset || 0); - } - - isTimeOffsetSet() { - return this.client.serverTimeOffset !== null; - } - _saveBasicOptions(authOptions: AuthOptions) { this.method = 'basic'; this.key = authOptions.key; @@ -913,7 +891,7 @@ class Auth { /* RSA4b1 -- if we have a server time offset set already, we can * automatically remove expired tokens. Else just use the cached token. If it is * expired Ably will tell us and we'll discard it then. */ - if (!this.isTimeOffsetSet() || !token.expires || token.expires >= this.getTimestampUsingOffset()) { + if (!this.client.isTimeOffsetSet() || !token.expires || token.expires >= this.client.getTimestampUsingOffset()) { Logger.logAction( this.logger, Logger.LOG_MINOR, @@ -1020,6 +998,13 @@ class Auth { ): Promise { return this.client.rest.revokeTokens(specifiers, options); } + + /** + * Same as {@link BaseClient.getTimestamp} but also takes into account {@link Auth.authOptions} + */ + private async _getTimestamp(queryTime: boolean): Promise { + return this.client.getTimestamp(queryTime || !!this.authOptions.queryTime); + } } export default Auth; diff --git a/src/common/lib/client/baseclient.ts b/src/common/lib/client/baseclient.ts index c79677b654..b9fa36da49 100644 --- a/src/common/lib/client/baseclient.ts +++ b/src/common/lib/client/baseclient.ts @@ -171,6 +171,28 @@ class BaseClient { this.logger.setLog(logOptions.level, logOptions.handler); } + /** + * Get the current time based on the local clock, + * or if the option queryTime is true, return the server time. + * The server time offset from the local time is stored so that + * only one request to the server to get the time is ever needed + */ + async getTimestamp(queryTime: boolean): Promise { + if (!this.isTimeOffsetSet() && queryTime) { + return this.time(); + } + + return this.getTimestampUsingOffset(); + } + + getTimestampUsingOffset(): number { + return Date.now() + (this.serverTimeOffset || 0); + } + + isTimeOffsetSet(): boolean { + return this.serverTimeOffset !== null; + } + static Platform = Platform; /** From 5b70e79db2a4d512164b4ea898c40dbee9251a1e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 01:06:05 +0000 Subject: [PATCH 101/166] Add object-level write API for LiveMap and LiveCounter creation Resolves DTP-1138 --- src/plugins/liveobjects/livecounter.ts | 65 ++++++++++++++++++++ src/plugins/liveobjects/livemap.ts | 84 ++++++++++++++++++++++++++ src/plugins/liveobjects/liveobject.ts | 5 -- src/plugins/liveobjects/liveobjects.ts | 60 ++++++++++++++++++ 4 files changed, 209 insertions(+), 5 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index a681f6087f..6c08f3d887 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -1,5 +1,6 @@ import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; +import { ObjectId } from './objectid'; import { StateCounterOp, StateMessage, StateObject, StateOperation, StateOperationAction } from './statemessage'; export interface LiveCounterData extends LiveObjectData { @@ -32,6 +33,18 @@ export class LiveCounter extends LiveObject return obj; } + /** + * Returns a {@link LiveCounter} instance based on the provided state operation. + * The provided state operation must hold a valid counter object data. + * + * @internal + */ + static fromStateOperation(liveobjects: LiveObjects, stateOperation: StateOperation): LiveCounter { + const obj = new LiveCounter(liveobjects, stateOperation.objectId); + obj._mergeInitialDataFromCreateOperation(stateOperation); + return obj; + } + /** * @internal */ @@ -57,6 +70,58 @@ export class LiveCounter extends LiveObject return stateMessage; } + /** + * @internal + */ + static async createCounterCreateMessage(liveObjects: LiveObjects, count?: number): Promise { + const client = liveObjects.getClient(); + + if (count !== undefined && (typeof count !== 'number' || !Number.isFinite(count))) { + throw new client.ErrorInfo('Counter value should be a valid number', 40013, 400); + } + + const initialValueObj = LiveCounter.createInitialValueObject(count); + const { encodedInitialValue, format } = StateMessage.encodeInitialValue(client.Utils, initialValueObj); + const nonce = client.Utils.cheapRandStr(); + const msTimestamp = await client.getTimestamp(true); + + const objectId = ObjectId.fromInitialValue( + client.Platform, + 'counter', + encodedInitialValue, + nonce, + msTimestamp, + ).toString(); + + const stateMessage = StateMessage.fromValues( + { + operation: { + ...initialValueObj, + action: StateOperationAction.COUNTER_CREATE, + objectId, + nonce, + initialValue: encodedInitialValue, + initialValueEncoding: format, + }, + }, + client.Utils, + client.MessageEncoding, + ); + + return stateMessage; + } + + /** + * @internal + */ + static createInitialValueObject(count?: number): Pick { + return { + counter: { + count: count ?? 0, + }, + }; + } + value(): number { return this._dataRef.data; } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 3a6efe90f5..8b3fc41efc 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -4,6 +4,7 @@ import type * as API from '../../../ably'; import { DEFAULTS } from './defaults'; import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { LiveObjects } from './liveobjects'; +import { ObjectId } from './objectid'; import { MapSemantics, StateMapEntry, @@ -80,6 +81,21 @@ export class LiveMap extends LiveObject( + liveobjects: LiveObjects, + stateOperation: StateOperation, + ): LiveMap { + const obj = new LiveMap(liveobjects, stateOperation.map?.semantics!, stateOperation.objectId); + obj._mergeInitialDataFromCreateOperation(stateOperation); + return obj; + } + /** * @internal */ @@ -170,6 +186,74 @@ export class LiveMap extends LiveObject { + const client = liveObjects.getClient(); + + if (entries !== undefined && (entries === null || typeof entries !== 'object')) { + throw new client.ErrorInfo('Map entries should be a key/value object', 40013, 400); + } + + Object.entries(entries ?? {}).forEach(([key, value]) => LiveMap.validateKeyValue(liveObjects, key, value)); + + const initialValueObj = LiveMap.createInitialValueObject(entries); + const { encodedInitialValue, format } = StateMessage.encodeInitialValue(client.Utils, initialValueObj); + const nonce = client.Utils.cheapRandStr(); + const msTimestamp = await client.getTimestamp(true); + + const objectId = ObjectId.fromInitialValue( + client.Platform, + 'map', + encodedInitialValue, + nonce, + msTimestamp, + ).toString(); + + const stateMessage = StateMessage.fromValues( + { + operation: { + ...initialValueObj, + action: StateOperationAction.MAP_CREATE, + objectId, + nonce, + initialValue: encodedInitialValue, + initialValueEncoding: format, + }, + }, + client.Utils, + client.MessageEncoding, + ); + + return stateMessage; + } + + /** + * @internal + */ + static createInitialValueObject(entries?: API.LiveMapType): Pick { + const stateMapEntries: Record = {}; + + Object.entries(entries ?? {}).forEach(([key, value]) => { + const stateData: StateData = + value instanceof LiveObject + ? ({ objectId: value.getObjectId() } as ObjectIdStateData) + : ({ value } as ValueStateData); + + stateMapEntries[key] = { + data: stateData, + }; + }); + + return { + map: { + semantics: MapSemantics.LWW, + entries: stateMapEntries, + }, + }; + } + /** * Returns the value associated with the specified key in the underlying Map object. * diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index 1d5f84bf18..2fe39596fa 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -161,11 +161,6 @@ export abstract class LiveObject< return this._updateFromDataDiff(previousDataRef, this._dataRef); } - private _createObjectId(): string { - // TODO: implement object id generation based on live object type and initial value - return Math.random().toString().substring(2); - } - /** * Apply state operation message on live object. * diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index c3c003c72c..0854a38ea9 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -54,6 +54,66 @@ export class LiveObjects { return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; } + /** + * Send a MAP_CREATE operation to the realtime system to create a new map object in the pool. + * + * Locally on the client it creates a zero-value object with the corresponding id and returns it. + * The object initialization with the initial value is expected to happen when the corresponding MAP_CREATE operation is echoed + * back to the client and applied to the object following the regular operation application procedure. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with a zero-value object created in the local pool. + */ + async createMap(entries?: T): Promise> { + const stateMessage = await LiveMap.createMapCreateMessage(this, entries); + const objectId = stateMessage.operation?.objectId!; + + await this.publish([stateMessage]); + + // we might have received CREATE operation already at this point (it might arrive before the ACK message for our publish message), + // so the object could exist in the local pool as it was added there during regular CREATE operation application. + // check if the object is there and return it in case it is. otherwise create a new object client-side + if (this._liveObjectsPool.get(objectId)) { + return this._liveObjectsPool.get(objectId) as LiveMap; + } + + // new map object can be created using locally constructed state operation, even though it is missing timeserials for map entries. + // CREATE operation is only applied once, and all map entries will have an "earliest possible" timeserial so that any subsequent operation can be applied to them. + const map = LiveMap.fromStateOperation(this, stateMessage.operation!); + this._liveObjectsPool.set(objectId, map); + + return map; + } + + /** + * Send a COUNTER_CREATE operation to the realtime system to create a new counter object in the pool. + * + * Locally on the client it creates a zero-value object with the corresponding id and returns it. + * The object initialization with the initial value is expected to happen when the corresponding COUNTER_CREATE operation is echoed + * back to the client and applied to the object following the regular operation application procedure. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with a zero-value object created in the local pool. + */ + async createCounter(count?: number): Promise { + const stateMessage = await LiveCounter.createCounterCreateMessage(this, count); + const objectId = stateMessage.operation?.objectId!; + + await this.publish([stateMessage]); + + // we might have received CREATE operation already at this point (it might arrive before the ACK message for our publish message), + // so the object could exist in the local pool as it was added there during regular CREATE operation application. + // check if the object is there and return it in case it is. otherwise create a new object client-side + if (this._liveObjectsPool.get(objectId)) { + return this._liveObjectsPool.get(objectId) as LiveCounter; + } + + // new counter object can be created using locally constructed state operation. + // CREATE operation is only applied once, so the initial counter value won't be double counted when we eventually receive an echoed CREATE operation + const counter = LiveCounter.fromStateOperation(this, stateMessage.operation!); + this._liveObjectsPool.set(objectId, counter); + + return counter; + } + /** * @internal */ From 77e0c3f75e37af248ebf7bcd72cb6e823a33462e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 05:02:48 +0000 Subject: [PATCH 102/166] Add tests for LiveMap/LiveCounter creation --- test/common/modules/private_api_recorder.js | 1 + test/realtime/live_objects.test.js | 444 +++++++++++++++++++- 2 files changed, 437 insertions(+), 8 deletions(-) diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index 74158c9109..13ce8f9c94 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -117,6 +117,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'read.transport.recvRequest.recvUri', 'read.transport.uri', 'replace.LiveObjects._liveObjectsPool._onGCInterval', + 'replace.LiveObjects.publish', 'replace.channel.attachImpl', 'replace.channel.processMessage', 'replace.channel.sendMessage', diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 0d60a6c194..b21acf1167 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -714,29 +714,29 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check created maps primitiveMapsFixtures.forEach((fixture) => { - const key = fixture.name; - const mapObj = root.get(key); + const mapKey = fixture.name; + const mapObj = root.get(mapKey); // check all maps exist on root - expect(mapObj, `Check map at "${key}" key in root exists`).to.exist; - expectInstanceOf(mapObj, 'LiveMap', `Check map at "${key}" key in root is of type LiveMap`); + expect(mapObj, `Check map at "${mapKey}" key in root exists`).to.exist; + expectInstanceOf(mapObj, 'LiveMap', `Check map at "${mapKey}" key in root is of type LiveMap`); // check primitive maps have correct values expect(mapObj.size()).to.equal( Object.keys(fixture.entries ?? {}).length, - `Check map "${key}" has correct number of keys`, + `Check map "${mapKey}" has correct number of keys`, ); Object.entries(fixture.entries ?? {}).forEach(([key, keyData]) => { if (keyData.data.encoding) { expect( BufferUtils.areBuffersEqual(mapObj.get(key), BufferUtils.base64Decode(keyData.data.value)), - `Check map "${key}" has correct value for "${key}" key`, + `Check map "${mapKey}" has correct value for "${key}" key`, ).to.be.true; } else { expect(mapObj.get(key)).to.equal( keyData.data.value, - `Check map "${key}" has correct value for "${key}" key`, + `Check map "${mapKey}" has correct value for "${key}" key`, ); } }); @@ -2425,6 +2425,434 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await expectRejectedWith(async () => map.remove(map), 'Map key should be string'); }, }, + + { + description: 'LiveObjects.createCounter sends COUNTER_CREATE operation', + action: async (ctx) => { + const { liveObjects } = ctx; + + const counters = await Promise.all(countersFixtures.map(async (x) => liveObjects.createCounter(x.count))); + + for (let i = 0; i < counters.length; i++) { + const counter = counters[i]; + const fixture = countersFixtures[i]; + + expect(counter, `Check counter #${i + 1} exists`).to.exist; + expectInstanceOf(counter, 'LiveCounter', `Check counter instance #${i + 1} is of an expected class`); + expect(counter.value()).to.equal( + fixture.count ?? 0, + `Check counter #${i + 1} has expected initial value`, + ); + } + }, + }, + + { + description: 'LiveCounter created with LiveObjects.createCounter can be assigned to the state tree', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + const counter = await liveObjects.createCounter(1); + await root.set('counter', counter); + + expectInstanceOf(counter, 'LiveCounter', `Check counter instance is of an expected class`); + expectInstanceOf( + root.get('counter'), + 'LiveCounter', + `Check counter instance on root is of an expected class`, + ); + expect(root.get('counter')).to.equal( + counter, + 'Check counter object on root is the same as from create method', + ); + expect(root.get('counter').value()).to.equal( + 1, + 'Check counter assigned to the state tree has the expected value', + ); + }, + }, + + { + description: + 'LiveObjects.createCounter can return LiveCounter with initial value without applying CREATE operation', + action: async (ctx) => { + const { liveObjects, helper } = ctx; + + // prevent publishing of ops to realtime so we guarantee that the initial value doesn't come from a CREATE op + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = () => {}; + + const counter = await liveObjects.createCounter(1); + expect(counter.value()).to.equal(1, `Check counter has expected initial value`); + }, + }, + + { + description: + 'LiveObjects.createCounter can return LiveCounter with initial value from applied CREATE operation', + action: async (ctx) => { + const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + + // instead of sending CREATE op to the realtime, echo it immediately to the client + // with forged initial value so we can check that counter gets initialized with a value from a CREATE op + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = async (stateMessages) => { + const counterId = stateMessages[0].operation.objectId; + // this should result in liveobjects' operation application procedure and create a object in the pool with forged initial value + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [liveObjectsHelper.counterCreateOp({ objectId: counterId, count: 10 })], + }); + }; + + const counter = await liveObjects.createCounter(1); + + // counter should be created with forged initial value instead of the actual one + expect(counter.value()).to.equal( + 10, + 'Check counter value has the expected initial value from a CREATE operation', + ); + }, + }, + + { + description: + 'Initial value is not double counted for LiveCounter from LiveObjects.createCounter when CREATE op is received', + action: async (ctx) => { + const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + + // prevent publishing of ops to realtime so we can guarantee order of operations + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = () => {}; + + // create counter locally, should have an initial value set + const counter = await liveObjects.createCounter(1); + helper.recordPrivateApi('call.LiveObject.getObjectId'); + const counterId = counter.getObjectId(); + + // now inject CREATE op for a counter with a forged value. it should not be applied + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [liveObjectsHelper.counterCreateOp({ objectId: counterId, count: 10 })], + }); + + expect(counter.value()).to.equal( + 1, + `Check counter initial value is not double counted after being created and receiving CREATE operation`, + ); + }, + }, + + { + description: 'LiveObjects.createCounter throws on invalid input', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + await expectRejectedWith( + async () => liveObjects.createCounter(null), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(Number.NaN), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(Number.POSITIVE_INFINITY), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(Number.NEGATIVE_INFINITY), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter('foo'), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(BigInt(1)), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(true), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(Symbol()), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter({}), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter([]), + 'Counter value should be a valid number', + ); + await expectRejectedWith( + async () => liveObjects.createCounter(root), + 'Counter value should be a valid number', + ); + }, + }, + + { + description: 'LiveObjects.createMap sends MAP_CREATE operation with primitive values', + action: async (ctx) => { + const { liveObjects } = ctx; + + const maps = await Promise.all( + primitiveMapsFixtures.map(async (mapFixture) => { + const entries = mapFixture.entries + ? Object.entries(mapFixture.entries).reduce((acc, [key, keyData]) => { + const value = keyData.data.encoding + ? BufferUtils.base64Decode(keyData.data.value) + : keyData.data.value; + acc[key] = value; + return acc; + }, {}) + : undefined; + + return liveObjects.createMap(entries); + }), + ); + + for (let i = 0; i < maps.length; i++) { + const map = maps[i]; + const fixture = primitiveMapsFixtures[i]; + + expect(map, `Check map #${i + 1} exists`).to.exist; + expectInstanceOf(map, 'LiveMap', `Check map instance #${i + 1} is of an expected class`); + + expect(map.size()).to.equal( + Object.keys(fixture.entries ?? {}).length, + `Check map #${i + 1} has correct number of keys`, + ); + + Object.entries(fixture.entries ?? {}).forEach(([key, keyData]) => { + if (keyData.data.encoding) { + expect( + BufferUtils.areBuffersEqual(map.get(key), BufferUtils.base64Decode(keyData.data.value)), + `Check map #${i + 1} has correct value for "${key}" key`, + ).to.be.true; + } else { + expect(map.get(key)).to.equal( + keyData.data.value, + `Check map #${i + 1} has correct value for "${key}" key`, + ); + } + }); + } + }, + }, + + { + description: 'LiveObjects.createMap sends MAP_CREATE operation with reference to another LiveObject', + action: async (ctx) => { + const { root, liveObjectsHelper, channelName, liveObjects } = ctx; + + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: liveObjectsHelper.counterCreateOp(), + }); + await liveObjectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: liveObjectsHelper.mapCreateOp(), + }); + + const counter = root.get('counter'); + const map = root.get('map'); + + const newMap = await liveObjects.createMap({ counter, map }); + + expect(newMap, 'Check map exists').to.exist; + expectInstanceOf(newMap, 'LiveMap', 'Check map instance is of an expected class'); + + expect(newMap.get('counter')).to.equal( + counter, + 'Check can set a reference to a LiveCounter object on a new map via a MAP_CREATE operation', + ); + expect(newMap.get('map')).to.equal( + map, + 'Check can set a reference to a LiveMap object on a new map via a MAP_CREATE operation', + ); + }, + }, + + { + description: 'LiveMap created with LiveObjects.createMap can be assigned to the state tree', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + const counter = await liveObjects.createCounter(); + const map = await liveObjects.createMap({ foo: 'bar', baz: counter }); + await root.set('map', map); + + expectInstanceOf(map, 'LiveMap', `Check map instance is of an expected class`); + expectInstanceOf(root.get('map'), 'LiveMap', `Check map instance on root is of an expected class`); + expect(root.get('map')).to.equal(map, 'Check map object on root is the same as from create method'); + expect(root.get('map').size()).to.equal( + 2, + 'Check map assigned to the state tree has the expected number of keys', + ); + expect(root.get('map').get('foo')).to.equal( + 'bar', + 'Check map assigned to the state tree has the expected value for its string key', + ); + expect(root.get('map').get('baz')).to.equal( + counter, + 'Check map assigned to the state tree has the expected value for its LiveCounter key', + ); + }, + }, + + { + description: 'LiveObjects.createMap can return LiveMap with initial value without applying CREATE operation', + action: async (ctx) => { + const { liveObjects, helper } = ctx; + + // prevent publishing of ops to realtime so we guarantee that the initial value doesn't come from a CREATE op + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = () => {}; + + const map = await liveObjects.createMap({ foo: 'bar' }); + expect(map.get('foo')).to.equal('bar', `Check map has expected initial value`); + }, + }, + + { + description: 'LiveObjects.createMap can return LiveMap with initial value from applied CREATE operation', + action: async (ctx) => { + const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + + // instead of sending CREATE op to the realtime, echo it immediately to the client + // with forged initial value so we can check that map gets initialized with a value from a CREATE op + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = async (stateMessages) => { + const mapId = stateMessages[0].operation.objectId; + // this should result in liveobjects' operation application procedure and create a object in the pool with forged initial value + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [ + liveObjectsHelper.mapCreateOp({ + objectId: mapId, + entries: { baz: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { value: 'qux' } } }, + }), + ], + }); + }; + + const map = await liveObjects.createMap({ foo: 'bar' }); + + // map should be created with forged initial value instead of the actual one + expect(map.get('foo'), `Check key "foo" was not set on a map client-side`).to.not.exist; + expect(map.get('baz')).to.equal( + 'qux', + `Check key "baz" was set on a map from a CREATE operation after object creation`, + ); + }, + }, + + { + description: + 'Initial value is not double counted for LiveMap from LiveObjects.createMap when CREATE op is received', + action: async (ctx) => { + const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + + // prevent publishing of ops to realtime so we can guarantee order of operations + helper.recordPrivateApi('replace.LiveObjects.publish'); + liveObjects.publish = () => {}; + + // create map locally, should have an initial value set + const map = await liveObjects.createMap({ foo: 'bar' }); + helper.recordPrivateApi('call.LiveObject.getObjectId'); + const mapId = map.getObjectId(); + + // now inject CREATE op for a map with a forged value. it should not be applied + await liveObjectsHelper.processStateOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [ + liveObjectsHelper.mapCreateOp({ + objectId: mapId, + entries: { + foo: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { value: 'qux' } }, + baz: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { value: 'qux' } }, + }, + }), + ], + }); + + expect(map.get('foo')).to.equal( + 'bar', + `Check key "foo" was not overridden by a CREATE operation after creating a map locally`, + ); + expect(map.get('baz'), `Check key "baz" was not set by a CREATE operation after creating a map locally`).to + .not.exist; + }, + }, + + { + description: 'LiveObjects.createMap throws on invalid input', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + await expectRejectedWith( + async () => liveObjects.createMap(null), + 'Map entries should be a key/value object', + ); + await expectRejectedWith( + async () => liveObjects.createMap('foo'), + 'Map entries should be a key/value object', + ); + await expectRejectedWith(async () => liveObjects.createMap(1), 'Map entries should be a key/value object'); + await expectRejectedWith( + async () => liveObjects.createMap(BigInt(1)), + 'Map entries should be a key/value object', + ); + await expectRejectedWith( + async () => liveObjects.createMap(true), + 'Map entries should be a key/value object', + ); + await expectRejectedWith( + async () => liveObjects.createMap(Symbol()), + 'Map entries should be a key/value object', + ); + + await expectRejectedWith( + async () => liveObjects.createMap({ key: undefined }), + 'Map value data type is unsupported', + ); + await expectRejectedWith( + async () => liveObjects.createMap({ key: null }), + 'Map value data type is unsupported', + ); + await expectRejectedWith( + async () => liveObjects.createMap({ key: BigInt(1) }), + 'Map value data type is unsupported', + ); + await expectRejectedWith( + async () => liveObjects.createMap({ key: Symbol() }), + 'Map value data type is unsupported', + ); + await expectRejectedWith( + async () => liveObjects.createMap({ key: {} }), + 'Map value data type is unsupported', + ); + await expectRejectedWith( + async () => liveObjects.createMap({ key: [] }), + 'Map value data type is unsupported', + ); + }, + }, ]; /** @nospec */ @@ -2447,7 +2875,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await channel.attach(); const root = await liveObjects.getRoot(); - await scenario.action({ root, liveObjectsHelper, channelName, channel }); + await scenario.action({ liveObjects, root, liveObjectsHelper, channelName, channel, client, helper }); }, client); }, ); From 4a33d69fc3dfdb9b8461a603657f6bed5d560669 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 06:33:47 +0000 Subject: [PATCH 103/166] Update minimal bundle size --- scripts/moduleReport.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index 8d1921789f..e1b67a8b2f 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -6,7 +6,7 @@ import { gzip } from 'zlib'; import Table from 'cli-table'; // The maximum size we allow for a minimal useful Realtime bundle (i.e. one that can subscribe to a channel) -const minimalUsefulRealtimeBundleSizeThresholdsKiB = { raw: 101, gzip: 31 }; +const minimalUsefulRealtimeBundleSizeThresholdsKiB = { raw: 102, gzip: 31 }; const baseClientNames = ['BaseRest', 'BaseRealtime']; From fcf5e54ee95b67c9d5f425cfe184ea2173aee93b Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 24 Jan 2025 04:04:44 +0000 Subject: [PATCH 104/166] Set channel serial when receiving `STATE` message See RTL15b and RTL4c1 Resolves DTP-1121 --- src/common/lib/client/realtimechannel.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index 427b6398e6..0a6d0cc528 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -516,7 +516,8 @@ class RealtimeChannel extends EventEmitter { if ( message.action === actions.ATTACHED || message.action === actions.MESSAGE || - message.action === actions.PRESENCE + message.action === actions.PRESENCE || + message.action === actions.STATE ) { // RTL15b this.setChannelSerial(message.channelSerial); From a7a160df58cfaa30641136d7f60acd6eb725ebe6 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 24 Jan 2025 04:05:48 +0000 Subject: [PATCH 105/166] Add tests for RTL4c1 and RTL15b --- test/realtime/channel.test.js | 90 +++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/test/realtime/channel.test.js b/test/realtime/channel.test.js index 92c1777a3e..1bdf114ee5 100644 --- a/test/realtime/channel.test.js +++ b/test/realtime/channel.test.js @@ -1861,5 +1861,95 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async await helper.closeAndFinishAsync(realtime); }); + + /** @spec RTL4c1 */ + it('set channelSerial field for ATTACH ProtocolMessage if available', async function () { + const helper = this.test.helper; + const realtime = helper.AblyRealtime(); + + await helper.monitorConnectionAsync(async () => { + const channel = realtime.channels.get('channel'); + channel.properties.channelSerial = 'channelSerial'; + + await realtime.connection.once('connected'); + + const promiseCheck = new Promise((resolve, reject) => { + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + const transport = realtime.connection.connectionManager.activeProtocol.getTransport(); + const sendOriginal = transport.send; + + helper.recordPrivateApi('replace.transport.send'); + transport.send = function (msg) { + if (msg.action === 10) { + try { + expect(msg.channelSerial).to.equal('channelSerial2'); + resolve(); + } catch (error) { + reject(error); + } + } else { + helper.recordPrivateApi('call.transport.send'); + sendOriginal.call(this, msg); + } + }; + }); + + // don't await for attach as it will never resolve in this test since we don't send ATTACH msg to realtime + channel.attach(); + await promiseCheck; + }, realtime); + + await helper.closeAndFinishAsync(realtime); + }); + + /** @spec RTL15b */ + it('channel.properties.channelSerial is updated with channelSerial from latest message', async function () { + const helper = this.test.helper; + const realtime = helper.AblyRealtime({ clientId: 'me' }); + + await helper.monitorConnectionAsync(async () => { + const channel = realtime.channels.get('channel'); + await realtime.connection.once('connected'); + + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + const messagesToUpdateChannelSerial = [ + createPM({ + action: 11, // ATTACHED + channel: channel.name, + channelSerial: 'ATTACHED', + }), + createPM({ + action: 15, // MESSAGE + channel: channel.name, + channelSerial: 'MESSAGE', + messages: [{ name: 'foo', data: 'bar' }], + }), + createPM({ + action: 14, // PRESENCE + channel: channel.name, + channelSerial: 'PRESENCE', + }), + createPM({ + action: 19, // STATE + channel: channel.name, + channelSerial: 'STATE', + }), + ]; + + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + const transport = realtime.connection.connectionManager.activeProtocol.getTransport(); + + for (const msg of messagesToUpdateChannelSerial) { + helper.recordPrivateApi('call.transport.onProtocolMessage'); + transport.onProtocolMessage(msg); + + // wait until next event loop so any async ops get resolved and channel serial gets updated on a channel + await new Promise((res) => setTimeout(res, 0)); + expect(channel.properties.channelSerial).to.equal(msg.channelSerial); + } + }, realtime); + + await helper.closeAndFinishAsync(realtime); + }); }); }); From ae655f18a4669ff44e8e79e6028059d99fd63c02 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 30 Jan 2025 01:23:28 +0000 Subject: [PATCH 106/166] Add `IBufferUtils.concat` method This is required for following LiveObjects changes to be able to encode the hash for object IDs based on multiple buffers. Browser implementation is based on example from [1]. [1] https://stackoverflow.com/questions/10786128/appending-arraybuffers --- src/common/types/IBufferUtils.ts | 1 + src/platform/nodejs/lib/util/bufferutils.ts | 4 ++++ src/platform/web/lib/util/bufferutils.ts | 15 +++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/src/common/types/IBufferUtils.ts b/src/common/types/IBufferUtils.ts index 8cd1d39aa3..8f9b8010bd 100644 --- a/src/common/types/IBufferUtils.ts +++ b/src/common/types/IBufferUtils.ts @@ -20,6 +20,7 @@ export default interface IBufferUtils { * Returns ArrayBuffer on browser and Buffer on Node.js */ arrayBufferViewToBuffer: (arrayBufferView: ArrayBufferView) => Bufferlike; + concat(buffers: Bufferlike[]): Output; sha256(message: Bufferlike): Output; hmacSha256(message: Bufferlike, key: Bufferlike): Output; } diff --git a/src/platform/nodejs/lib/util/bufferutils.ts b/src/platform/nodejs/lib/util/bufferutils.ts index 8c93f4ef34..82ba2f2875 100644 --- a/src/platform/nodejs/lib/util/bufferutils.ts +++ b/src/platform/nodejs/lib/util/bufferutils.ts @@ -74,6 +74,10 @@ class BufferUtils implements IBufferUtils { return Buffer.from(string, 'utf8'); } + concat(buffers: Bufferlike[]): Output { + return Buffer.concat(buffers.map((x) => this.toBuffer(x))); + } + sha256(message: Bufferlike): Output { const messageBuffer = this.toBuffer(message); diff --git a/src/platform/web/lib/util/bufferutils.ts b/src/platform/web/lib/util/bufferutils.ts index 1d7af7d694..cccd538c78 100644 --- a/src/platform/web/lib/util/bufferutils.ts +++ b/src/platform/web/lib/util/bufferutils.ts @@ -200,6 +200,21 @@ class BufferUtils implements IBufferUtils { return this.toArrayBuffer(arrayBufferView); } + concat(buffers: Bufferlike[]): Output { + const sumLength = buffers.reduce((acc, v) => acc + v.byteLength, 0); + const result = new Uint8Array(sumLength); + let offset = 0; + + for (const buffer of buffers) { + const uint8Array = this.toBuffer(buffer); + // see TypedArray.set for TypedArray argument https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray/set#typedarray + result.set(uint8Array, offset); + offset += uint8Array.byteLength; + } + + return result.buffer; + } + sha256(message: Bufferlike): Output { const hash = sha256(this.toBuffer(message)); return this.toArrayBuffer(hash); From 109fefb4e7eefe4bbc084703afc1f62030fc66c7 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 30 Jan 2025 01:36:05 +0000 Subject: [PATCH 107/166] Encode `initialValue` for StateMessage based on the `useBinaryProtocol` client option This adds support for encoding `initialValue` for `json` and `msgpack` encodings. Also improves types in StateMessage, notably use `Bufferlike` instead of `Buffer | ArrayBuffer`. --- src/plugins/liveobjects/livecounter.ts | 6 +-- src/plugins/liveobjects/livemap.ts | 8 +-- src/plugins/liveobjects/objectid.ts | 9 +++- src/plugins/liveobjects/statemessage.ts | 71 +++++++++++++++---------- 4 files changed, 56 insertions(+), 38 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 6c08f3d887..8d8d8e932e 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -61,7 +61,7 @@ export class LiveCounter extends LiveObject action: StateOperationAction.COUNTER_INC, objectId, counterOp: { amount }, - }, + } as StateOperation, }, client.Utils, client.MessageEncoding, @@ -81,7 +81,7 @@ export class LiveCounter extends LiveObject } const initialValueObj = LiveCounter.createInitialValueObject(count); - const { encodedInitialValue, format } = StateMessage.encodeInitialValue(client.Utils, initialValueObj); + const { encodedInitialValue, format } = StateMessage.encodeInitialValue(initialValueObj, client); const nonce = client.Utils.cheapRandStr(); const msTimestamp = await client.getTimestamp(true); @@ -102,7 +102,7 @@ export class LiveCounter extends LiveObject nonce, initialValue: encodedInitialValue, initialValueEncoding: format, - }, + } as StateOperation, }, client.Utils, client.MessageEncoding, diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 8b3fc41efc..9d35b868bf 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -123,7 +123,7 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject LiveMap.validateKeyValue(liveObjects, key, value)); const initialValueObj = LiveMap.createInitialValueObject(entries); - const { encodedInitialValue, format } = StateMessage.encodeInitialValue(client.Utils, initialValueObj); + const { encodedInitialValue, format } = StateMessage.encodeInitialValue(initialValueObj, client); const nonce = client.Utils.cheapRandStr(); const msTimestamp = await client.getTimestamp(true); @@ -220,7 +220,7 @@ export class LiveMap extends LiveObject { value: StateValue | undefined; encoding: string | undefined }; +export type EncodeFunction = (data: any, encoding?: string | null) => { data: any; encoding?: string | null }; export enum StateOperationAction { MAP_CREATE = 0, @@ -21,7 +20,7 @@ export enum MapSemantics { } /** A StateValue represents a concrete leaf value in a state object graph. */ -export type StateValue = string | number | boolean | Buffer | ArrayBuffer; +export type StateValue = string | number | boolean | Bufferlike; /** StateData captures a value in a state object. */ export interface StateData { @@ -110,9 +109,9 @@ export interface StateOperation { * After verification the bytes will be decoded into the Map or Counter objects and * the initialValue, nonce, and initialValueEncoding will be removed. */ - initialValue?: Buffer | ArrayBuffer; + initialValue?: Bufferlike; /** The initial value encoding defines how the initialValue should be interpreted. */ - initialValueEncoding?: string; + initialValueEncoding?: Utils.Format; } /** A StateObject describes the instantaneous state of an object. */ @@ -172,12 +171,12 @@ export class StateMessage { * Uses encoding functions from regular `Message` processing. */ static async encode(message: StateMessage, messageEncoding: typeof MessageEncoding): Promise { - const encodeFn: StateDataEncodeFunction = (value, encoding) => { - const { data: newValue, encoding: newEncoding } = messageEncoding.encodeData(value, encoding); + const encodeFn: EncodeFunction = (data, encoding) => { + const { data: encodedData, encoding: newEncoding } = messageEncoding.encodeData(data, encoding); return { - value: newValue, - encoding: newEncoding!, + data: encodedData, + encoding: newEncoding, }; }; @@ -242,15 +241,26 @@ export class StateMessage { } static encodeInitialValue( - utils: typeof Utils, initialValue: Partial, + client: BaseClient, ): { - encodedInitialValue: string; + encodedInitialValue: Bufferlike; format: Utils.Format; } { + const format = client.options.useBinaryProtocol ? client.Utils.Format.msgpack : client.Utils.Format.json; + const encodedInitialValue = client.Utils.encodeBody(initialValue, client._MsgPack, format); + + // if we've got string result (for example, json format was used), we need to additionally convert it to bytes array with utf8 encoding + if (typeof encodedInitialValue === 'string') { + return { + encodedInitialValue: client.Platform.BufferUtils.utf8Encode(encodedInitialValue), + format, + }; + } + return { - encodedInitialValue: JSON.stringify(initialValue), - format: utils.Format.json, + encodedInitialValue, + format, }; } @@ -282,10 +292,7 @@ export class StateMessage { } } - private static _encodeStateOperation( - stateOperation: StateOperation, - encodeFn: StateDataEncodeFunction, - ): StateOperation { + private static _encodeStateOperation(stateOperation: StateOperation, encodeFn: EncodeFunction): StateOperation { // deep copy "stateOperation" object so we can modify the copy here. // buffer values won't be correctly copied, so we will need to set them again explicitly. const stateOperationCopy = JSON.parse(JSON.stringify(stateOperation)) as StateOperation; @@ -302,10 +309,16 @@ export class StateMessage { }); } + if (stateOperation.initialValue) { + // use original "stateOperation" object so we have access to the original buffer value + const { data: encodedInitialValue } = encodeFn(stateOperation.initialValue); + stateOperationCopy.initialValue = encodedInitialValue; + } + return stateOperationCopy; } - private static _encodeStateObject(stateObject: StateObject, encodeFn: StateDataEncodeFunction): StateObject { + private static _encodeStateObject(stateObject: StateObject, encodeFn: EncodeFunction): StateObject { // deep copy "stateObject" object so we can modify the copy here. // buffer values won't be correctly copied, so we will need to set them again explicitly. const stateObjectCopy = JSON.parse(JSON.stringify(stateObject)) as StateObject; @@ -325,13 +338,13 @@ export class StateMessage { return stateObjectCopy; } - private static _encodeStateData(data: StateData, encodeFn: StateDataEncodeFunction): StateData { - const { value: newValue, encoding: newEncoding } = encodeFn(data?.value, data?.encoding); + private static _encodeStateData(data: StateData, encodeFn: EncodeFunction): StateData { + const { data: encodedValue, encoding: newEncoding } = encodeFn(data?.value, data?.encoding); return { ...data, - value: newValue, - encoding: newEncoding!, + value: encodedValue, + encoding: newEncoding ?? undefined, }; } @@ -353,15 +366,15 @@ export class StateMessage { // if JSON protocol is being used, the JSON.stringify() will be called and this toJSON() method will have a non-empty arguments list. // MSGPack protocol implementation also calls toJSON(), but with an empty arguments list. const format = arguments.length > 0 ? this._utils.Format.json : this._utils.Format.msgpack; - const encodeFn: StateDataEncodeFunction = (value, encoding) => { - const { data: newValue, encoding: newEncoding } = this._messageEncoding.encodeDataForWireProtocol( - value, + const encodeFn: EncodeFunction = (data, encoding) => { + const { data: encodedData, encoding: newEncoding } = this._messageEncoding.encodeDataForWireProtocol( + data, encoding, format, ); return { - value: newValue, - encoding: newEncoding!, + data: encodedData, + encoding: newEncoding, }; }; From 6ea31dacf4fea78ffd0f175352e14e1a3ffff33f Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 30 Jan 2025 02:35:22 +0000 Subject: [PATCH 108/166] Fix `StateMessage.encodeInitialValue` incorrectly encoded initial value with `json` encoding when it contained buffers for LiveMap keys --- src/plugins/liveobjects/statemessage.ts | 71 ++++++++++++++++++------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 0bf2e148e9..77fdc5cba8 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -248,9 +248,21 @@ export class StateMessage { format: Utils.Format; } { const format = client.options.useBinaryProtocol ? client.Utils.Format.msgpack : client.Utils.Format.json; - const encodedInitialValue = client.Utils.encodeBody(initialValue, client._MsgPack, format); - // if we've got string result (for example, json format was used), we need to additionally convert it to bytes array with utf8 encoding + // initial value object may contain user provided data that requires an additional encoding (for example buffers as map keys). + // so we need to encode that data first as if we were sending it over the wire. we can use a StateMessage methods for this + const stateMessage = StateMessage.fromValues({ operation: initialValue }, client.Utils, client.MessageEncoding); + StateMessage.encode(stateMessage, client.MessageEncoding); + const { operation: initialValueWithDataEncoding } = StateMessage._encodeForWireProtocol( + stateMessage, + client.MessageEncoding, + format, + ); + + // initial value field should be represented as an array of bytes over the wire. so we encode the whole object based on the client encoding format + const encodedInitialValue = client.Utils.encodeBody(initialValueWithDataEncoding, client._MsgPack, format); + + // if we've got string result (for example, json encoding was used), we need to additionally convert it to bytes array with utf8 encoding if (typeof encodedInitialValue === 'string') { return { encodedInitialValue: client.Platform.BufferUtils.utf8Encode(encodedInitialValue), @@ -348,6 +360,42 @@ export class StateMessage { }; } + /** + * Encodes operation and object fields of the StateMessage. Does not mutate the provided StateMessage. + * + * Uses encoding functions from regular `Message` processing. + */ + private static _encodeForWireProtocol( + message: StateMessage, + messageEncoding: typeof MessageEncoding, + format: Utils.Format, + ): { + operation?: StateOperation; + object?: StateObject; + } { + const encodeFn: EncodeFunction = (data, encoding) => { + const { data: encodedData, encoding: newEncoding } = messageEncoding.encodeDataForWireProtocol( + data, + encoding, + format, + ); + return { + data: encodedData, + encoding: newEncoding, + }; + }; + + const encodedOperation = message.operation + ? StateMessage._encodeStateOperation(message.operation, encodeFn) + : undefined; + const encodedObject = message.object ? StateMessage._encodeStateObject(message.object, encodeFn) : undefined; + + return { + operation: encodedOperation, + object: encodedObject, + }; + } + /** * Overload toJSON() to intercept JSON.stringify(). * @@ -366,26 +414,13 @@ export class StateMessage { // if JSON protocol is being used, the JSON.stringify() will be called and this toJSON() method will have a non-empty arguments list. // MSGPack protocol implementation also calls toJSON(), but with an empty arguments list. const format = arguments.length > 0 ? this._utils.Format.json : this._utils.Format.msgpack; - const encodeFn: EncodeFunction = (data, encoding) => { - const { data: encodedData, encoding: newEncoding } = this._messageEncoding.encodeDataForWireProtocol( - data, - encoding, - format, - ); - return { - data: encodedData, - encoding: newEncoding, - }; - }; - - const encodedOperation = this.operation ? StateMessage._encodeStateOperation(this.operation, encodeFn) : undefined; - const encodedObject = this.object ? StateMessage._encodeStateObject(this.object, encodeFn) : undefined; + const { operation, object } = StateMessage._encodeForWireProtocol(this, this._messageEncoding, format); return { id: this.id, clientId: this.clientId, - operation: encodedOperation, - object: encodedObject, + operation, + object, extras: this.extras, }; } From 5cfbc9b130eb35e3ef7fc77008afc8bd6243edbd Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 31 Jan 2025 01:57:23 +0000 Subject: [PATCH 109/166] Improve comments about counter/map creation --- src/plugins/liveobjects/livecounter.ts | 2 +- src/plugins/liveobjects/livemap.ts | 2 +- src/plugins/liveobjects/liveobjects.ts | 21 +++++++++++---------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 8d8d8e932e..05425ff626 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -34,7 +34,7 @@ export class LiveCounter extends LiveObject } /** - * Returns a {@link LiveCounter} instance based on the provided state operation. + * Returns a {@link LiveCounter} instance based on the provided COUNTER_CREATE state operation. * The provided state operation must hold a valid counter object data. * * @internal diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 9d35b868bf..f983c52e31 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -82,7 +82,7 @@ export class LiveMap extends LiveObject; } - // new map object can be created using locally constructed state operation, even though it is missing timeserials for map entries. - // CREATE operation is only applied once, and all map entries will have an "earliest possible" timeserial so that any subsequent operation can be applied to them. + // we haven't received the CREATE operation yet, so we can create a new map object using the locally constructed state operation. + // we don't know the timeserials for map entries, so we assign an "earliest possible" timeserial to each entry, so that any subsequent operation can be applied to them. + // we mark the CREATE operation as merged for the object, guaranteeing its idempotency and preventing it from being applied again when the operation arrives. const map = LiveMap.fromStateOperation(this, stateMessage.operation!); this._liveObjectsPool.set(objectId, map); @@ -99,15 +100,15 @@ export class LiveObjects { await this.publish([stateMessage]); - // we might have received CREATE operation already at this point (it might arrive before the ACK message for our publish message), - // so the object could exist in the local pool as it was added there during regular CREATE operation application. - // check if the object is there and return it in case it is. otherwise create a new object client-side + // we may have already received the CREATE operation at this point, as it could arrive before the ACK for our publish message. + // this means the object might already exist in the local pool, having been added during the usual CREATE operation process. + // here we check if the object is present, and return it if found; otherwise, create a new object on the client side. if (this._liveObjectsPool.get(objectId)) { return this._liveObjectsPool.get(objectId) as LiveCounter; } - // new counter object can be created using locally constructed state operation. - // CREATE operation is only applied once, so the initial counter value won't be double counted when we eventually receive an echoed CREATE operation + // we haven't received the CREATE operation yet, so we can create a new counter object using the locally constructed state operation. + // we mark the CREATE operation as merged for the object, guaranteeing its idempotency. this ensures we don't double count the initial counter value when the operation arrives. const counter = LiveCounter.fromStateOperation(this, stateMessage.operation!); this._liveObjectsPool.set(objectId, counter); From e031f012bd88a893a1c3cfa4561cead95797c983 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 16 Jan 2025 06:26:34 +0000 Subject: [PATCH 110/166] Add Batch write API for LiveObjects Resolves DTP-1035 --- scripts/moduleReport.ts | 3 + src/plugins/liveobjects/batchcontext.ts | 106 +++++++ .../liveobjects/batchcontextlivecounter.ts | 38 +++ .../liveobjects/batchcontextlivemap.ts | 40 +++ src/plugins/liveobjects/liveobjects.ts | 20 +- test/realtime/live_objects.test.js | 276 ++++++++++++++++++ 6 files changed, 482 insertions(+), 1 deletion(-) create mode 100644 src/plugins/liveobjects/batchcontext.ts create mode 100644 src/plugins/liveobjects/batchcontextlivecounter.ts create mode 100644 src/plugins/liveobjects/batchcontextlivemap.ts diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index e1b67a8b2f..28c440ccbe 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -323,6 +323,9 @@ async function checkLiveObjectsPluginFiles() { // These are the files that are allowed to contribute >= `threshold` bytes to the LiveObjects bundle. const allowedFiles = new Set([ + 'src/plugins/liveobjects/batchcontext.ts', + 'src/plugins/liveobjects/batchcontextlivecounter.ts', + 'src/plugins/liveobjects/batchcontextlivemap.ts', 'src/plugins/liveobjects/index.ts', 'src/plugins/liveobjects/livecounter.ts', 'src/plugins/liveobjects/livemap.ts', diff --git a/src/plugins/liveobjects/batchcontext.ts b/src/plugins/liveobjects/batchcontext.ts new file mode 100644 index 0000000000..db0dcba34f --- /dev/null +++ b/src/plugins/liveobjects/batchcontext.ts @@ -0,0 +1,106 @@ +import type BaseClient from 'common/lib/client/baseclient'; +import type * as API from '../../../ably'; +import { BatchContextLiveCounter } from './batchcontextlivecounter'; +import { BatchContextLiveMap } from './batchcontextlivemap'; +import { LiveCounter } from './livecounter'; +import { LiveMap } from './livemap'; +import { LiveObjects } from './liveobjects'; +import { ROOT_OBJECT_ID } from './liveobjectspool'; +import { StateMessage } from './statemessage'; + +export class BatchContext { + private _client: BaseClient; + /** Maps object ids to the corresponding batch context object wrappers for Live Objects in the pool */ + private _wrappedObjects: Map> = new Map(); + private _queuedMessages: StateMessage[] = []; + private _isClosed = false; + + constructor( + private _liveObjects: LiveObjects, + private _root: LiveMap, + ) { + this._client = _liveObjects.getClient(); + this._wrappedObjects.set(this._root.getObjectId(), new BatchContextLiveMap(this, this._liveObjects, this._root)); + } + + getRoot(): BatchContextLiveMap { + this.throwIfClosed(); + return this.getWrappedObject(ROOT_OBJECT_ID) as BatchContextLiveMap; + } + + /** + * @internal + */ + getWrappedObject(objectId: string): BatchContextLiveCounter | BatchContextLiveMap | undefined { + if (this._wrappedObjects.has(objectId)) { + return this._wrappedObjects.get(objectId); + } + + const originObject = this._liveObjects.getPool().get(objectId); + if (!originObject) { + return undefined; + } + + let wrappedObject: BatchContextLiveCounter | BatchContextLiveMap; + if (originObject instanceof LiveMap) { + wrappedObject = new BatchContextLiveMap(this, this._liveObjects, originObject); + } else if (originObject instanceof LiveCounter) { + wrappedObject = new BatchContextLiveCounter(this, this._liveObjects, originObject); + } else { + throw new this._client.ErrorInfo( + `Unknown Live Object instance type: objectId=${originObject.getObjectId()}`, + 50000, + 500, + ); + } + + this._wrappedObjects.set(objectId, wrappedObject); + return wrappedObject; + } + + /** + * @internal + */ + throwIfClosed(): void { + if (this.isClosed()) { + throw new this._client.ErrorInfo('Batch is closed', 40000, 400); + } + } + + /** + * @internal + */ + isClosed(): boolean { + return this._isClosed; + } + + /** + * @internal + */ + close(): void { + this._isClosed = true; + } + + /** + * @internal + */ + queueStateMessage(stateMessage: StateMessage): void { + this._queuedMessages.push(stateMessage); + } + + /** + * @internal + */ + async flush(): Promise { + try { + this.close(); + + if (this._queuedMessages.length > 0) { + await this._liveObjects.publish(this._queuedMessages); + } + } finally { + this._wrappedObjects.clear(); + this._queuedMessages = []; + } + } +} diff --git a/src/plugins/liveobjects/batchcontextlivecounter.ts b/src/plugins/liveobjects/batchcontextlivecounter.ts new file mode 100644 index 0000000000..5462fada2f --- /dev/null +++ b/src/plugins/liveobjects/batchcontextlivecounter.ts @@ -0,0 +1,38 @@ +import type BaseClient from 'common/lib/client/baseclient'; +import { BatchContext } from './batchcontext'; +import { LiveCounter } from './livecounter'; +import { LiveObjects } from './liveobjects'; + +export class BatchContextLiveCounter { + private _client: BaseClient; + + constructor( + private _batchContext: BatchContext, + private _liveObjects: LiveObjects, + private _counter: LiveCounter, + ) { + this._client = this._liveObjects.getClient(); + } + + value(): number { + this._batchContext.throwIfClosed(); + return this._counter.value(); + } + + increment(amount: number): void { + this._batchContext.throwIfClosed(); + const stateMessage = LiveCounter.createCounterIncMessage(this._liveObjects, this._counter.getObjectId(), amount); + this._batchContext.queueStateMessage(stateMessage); + } + + decrement(amount: number): void { + this._batchContext.throwIfClosed(); + // do an explicit type safety check here before negating the amount value, + // so we don't unintentionally change the type sent by a user + if (typeof amount !== 'number') { + throw new this._client.ErrorInfo('Counter value decrement should be a number', 40013, 400); + } + + this.increment(-amount); + } +} diff --git a/src/plugins/liveobjects/batchcontextlivemap.ts b/src/plugins/liveobjects/batchcontextlivemap.ts new file mode 100644 index 0000000000..0c0a84fc7d --- /dev/null +++ b/src/plugins/liveobjects/batchcontextlivemap.ts @@ -0,0 +1,40 @@ +import type * as API from '../../../ably'; +import { BatchContext } from './batchcontext'; +import { LiveMap } from './livemap'; +import { LiveObject } from './liveobject'; +import { LiveObjects } from './liveobjects'; + +export class BatchContextLiveMap { + constructor( + private _batchContext: BatchContext, + private _liveObjects: LiveObjects, + private _map: LiveMap, + ) {} + + get(key: TKey): T[TKey] | undefined { + this._batchContext.throwIfClosed(); + const value = this._map.get(key); + if (value instanceof LiveObject) { + return this._batchContext.getWrappedObject(value.getObjectId()) as T[TKey]; + } else { + return value; + } + } + + size(): number { + this._batchContext.throwIfClosed(); + return this._map.size(); + } + + set(key: TKey, value: T[TKey]): void { + this._batchContext.throwIfClosed(); + const stateMessage = LiveMap.createMapSetMessage(this._liveObjects, this._map.getObjectId(), key, value); + this._batchContext.queueStateMessage(stateMessage); + } + + remove(key: TKey): void { + this._batchContext.throwIfClosed(); + const stateMessage = LiveMap.createMapRemoveMessage(this._liveObjects, this._map.getObjectId(), key); + this._batchContext.queueStateMessage(stateMessage); + } +} diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index d463f13dfc..1ae68e1cba 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -2,6 +2,7 @@ import type BaseClient from 'common/lib/client/baseclient'; import type RealtimeChannel from 'common/lib/client/realtimechannel'; import type EventEmitter from 'common/lib/util/eventemitter'; import type * as API from '../../../ably'; +import { BatchContext } from './batchcontext'; import { DEFAULTS } from './defaults'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; @@ -14,6 +15,8 @@ enum LiveObjectsEvents { SyncCompleted = 'SyncCompleted', } +type BatchCallback = (batchContext: BatchContext) => void; + export class LiveObjects { private _client: BaseClient; private _channel: RealtimeChannel; @@ -54,6 +57,21 @@ export class LiveObjects { return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; } + /** + * Provides access to the synchronous write API for LiveObjects that can be used to batch multiple operations together in a single channel message. + */ + async batch(callback: BatchCallback): Promise { + const root = await this.getRoot(); + const context = new BatchContext(this, root); + + try { + callback(context); + await context.flush(); + } finally { + context.close(); + } + } + /** * Send a MAP_CREATE operation to the realtime system to create a new map object in the pool. * @@ -302,7 +320,7 @@ export class LiveObjects { break; default: - throw new this._client.ErrorInfo(`Unknown live object type: ${objectType}`, 50000, 500); + throw new this._client.ErrorInfo(`Unknown Live Object type: ${objectType}`, 50000, 500); } this._liveObjectsPool.set(objectId, newObject); diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index b21acf1167..db76ea78cb 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -2853,6 +2853,282 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ); }, }, + + { + description: 'batch API getRoot method is synchronous', + action: async (ctx) => { + const { liveObjects } = ctx; + + await liveObjects.batch((ctx) => { + const root = ctx.getRoot(); + expect(root, 'Check getRoot method in a BatchContext returns root object synchronously').to.exist; + expectInstanceOf(root, 'LiveMap', 'root object obtained from a BatchContext is a LiveMap'); + }); + }, + }, + + { + description: 'batch API .get method on a map returns BatchContext* wrappers for live objects', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + const counter = await liveObjects.createCounter(1); + const map = await liveObjects.createMap({ innerCounter: counter }); + await root.set('counter', counter); + await root.set('map', map); + + await liveObjects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + const ctxCounter = ctxRoot.get('counter'); + const ctxMap = ctxRoot.get('map'); + const ctxInnerCounter = ctxMap.get('innerCounter'); + + expect(ctxCounter, 'Check counter object can be accessed from a map in a batch API').to.exist; + expectInstanceOf( + ctxCounter, + 'BatchContextLiveCounter', + 'Check counter object obtained in a batch API has a BatchContext specific wrapper type', + ); + expect(ctxMap, 'Check map object can be accessed from a map in a batch API').to.exist; + expectInstanceOf( + ctxMap, + 'BatchContextLiveMap', + 'Check map object obtained in a batch API has a BatchContext specific wrapper type', + ); + expect(ctxInnerCounter, 'Check inner counter object can be accessed from a map in a batch API').to.exist; + expectInstanceOf( + ctxInnerCounter, + 'BatchContextLiveCounter', + 'Check inner counter object obtained in a batch API has a BatchContext specific wrapper type', + ); + }); + }, + }, + + { + description: 'batch API access API methods on live objects work and are synchronous', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + const counter = await liveObjects.createCounter(1); + const map = await liveObjects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + + await liveObjects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + const ctxCounter = ctxRoot.get('counter'); + const ctxMap = ctxRoot.get('map'); + + expect(ctxCounter.value()).to.equal( + 1, + 'Check batch API counter .value() method works and is synchronous', + ); + expect(ctxMap.get('foo')).to.equal('bar', 'Check batch API map .get() method works and is synchronous'); + expect(ctxMap.size()).to.equal(1, 'Check batch API map .size() method works and is synchronous'); + }); + }, + }, + + { + description: 'batch API write API methods on live objects do not mutate objects inside the batch callback', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + const counter = await liveObjects.createCounter(1); + const map = await liveObjects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + + await liveObjects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + const ctxCounter = ctxRoot.get('counter'); + const ctxMap = ctxRoot.get('map'); + + ctxCounter.increment(10); + expect(ctxCounter.value()).to.equal( + 1, + 'Check batch API counter .increment method does not mutate the object inside the batch callback', + ); + + ctxCounter.decrement(100); + expect(ctxCounter.value()).to.equal( + 1, + 'Check batch API counter .decrement method does not mutate the object inside the batch callback', + ); + + ctxMap.set('baz', 'qux'); + expect( + ctxMap.get('baz'), + 'Check batch API map .set method does not mutate the object inside the batch callback', + ).to.not.exist; + + ctxMap.remove('foo'); + expect(ctxMap.get('foo')).to.equal( + 'bar', + 'Check batch API map .remove method does not mutate the object inside the batch callback', + ); + }); + }, + }, + + { + description: 'batch API scheduled operations are applied when batch callback is finished', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + const counter = await liveObjects.createCounter(1); + const map = await liveObjects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + + await liveObjects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + const ctxCounter = ctxRoot.get('counter'); + const ctxMap = ctxRoot.get('map'); + + ctxCounter.increment(10); + ctxCounter.decrement(100); + + ctxMap.set('baz', 'qux'); + ctxMap.remove('foo'); + }); + + expect(counter.value()).to.equal(1 + 10 - 100, 'Check counter has an expected value after batch call'); + expect(map.get('baz')).to.equal('qux', 'Check key "baz" has an expected value in a map after batch call'); + expect(map.get('foo'), 'Check key "foo" is removed from map after batch call').to.not.exist; + }, + }, + + { + description: 'batch API can be called without scheduling any operations', + action: async (ctx) => { + const { liveObjects } = ctx; + + let caughtError; + try { + await liveObjects.batch((ctx) => {}); + } catch (error) { + caughtError = error; + } + expect( + caughtError, + `Check batch API can be called without scheduling any operations, but got error: ${caughtError?.toString()}`, + ).to.not.exist; + }, + }, + + { + description: 'batch API scheduled operations can be canceled by throwing an error in the batch callback', + action: async (ctx) => { + const { root, liveObjects } = ctx; + + const counter = await liveObjects.createCounter(1); + const map = await liveObjects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + + const cancelError = new Error('cancel batch'); + let caughtError; + try { + await liveObjects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + const ctxCounter = ctxRoot.get('counter'); + const ctxMap = ctxRoot.get('map'); + + ctxCounter.increment(10); + ctxCounter.decrement(100); + + ctxMap.set('baz', 'qux'); + ctxMap.remove('foo'); + + throw cancelError; + }); + } catch (error) { + caughtError = error; + } + + expect(counter.value()).to.equal(1, 'Check counter value is not changed after canceled batch call'); + expect(map.get('baz'), 'Check key "baz" does not exist on a map after canceled batch call').to.not.exist; + expect(map.get('foo')).to.equal('bar', 'Check key "foo" is not changed on a map after canceled batch call'); + expect(caughtError).to.equal( + cancelError, + 'Check error from a batch callback was rethrown by a batch method', + ); + }, + }, + + { + description: `batch API batch context and derived objects can't be interacted with after the batch call`, + action: async (ctx) => { + const { root, liveObjects } = ctx; + + const counter = await liveObjects.createCounter(1); + const map = await liveObjects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + + let savedCtx; + let savedCtxCounter; + let savedCtxMap; + + await liveObjects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + savedCtx = ctx; + savedCtxCounter = ctxRoot.get('counter'); + savedCtxMap = ctxRoot.get('map'); + }); + + expect(() => savedCtx.getRoot()).to.throw('Batch is closed'); + expect(() => savedCtxCounter.value()).to.throw('Batch is closed'); + expect(() => savedCtxCounter.increment()).to.throw('Batch is closed'); + expect(() => savedCtxCounter.decrement()).to.throw('Batch is closed'); + expect(() => savedCtxMap.get()).to.throw('Batch is closed'); + expect(() => savedCtxMap.size()).to.throw('Batch is closed'); + expect(() => savedCtxMap.set()).to.throw('Batch is closed'); + expect(() => savedCtxMap.remove()).to.throw('Batch is closed'); + }, + }, + + { + description: `batch API batch context and derived objects can't be interacted with thrown error from batch callback`, + action: async (ctx) => { + const { root, liveObjects } = ctx; + + const counter = await liveObjects.createCounter(1); + const map = await liveObjects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + + let savedCtx; + let savedCtxCounter; + let savedCtxMap; + + let caughtError; + try { + await liveObjects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + savedCtx = ctx; + savedCtxCounter = ctxRoot.get('counter'); + savedCtxMap = ctxRoot.get('map'); + + throw new Error('cancel batch'); + }); + } catch (error) { + caughtError = error; + } + + expect(caughtError, 'Check batch call failed with an error').to.exist; + expect(() => savedCtx.getRoot()).to.throw('Batch is closed'); + expect(() => savedCtxCounter.value()).to.throw('Batch is closed'); + expect(() => savedCtxCounter.increment()).to.throw('Batch is closed'); + expect(() => savedCtxCounter.decrement()).to.throw('Batch is closed'); + expect(() => savedCtxMap.get()).to.throw('Batch is closed'); + expect(() => savedCtxMap.size()).to.throw('Batch is closed'); + expect(() => savedCtxMap.set()).to.throw('Batch is closed'); + expect(() => savedCtxMap.remove()).to.throw('Batch is closed'); + }, + }, ]; /** @nospec */ From 53d07ffb2ad1595033583a6c7cfe7f83e5a18ea5 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 22 Jan 2025 04:52:46 +0000 Subject: [PATCH 111/166] Surface client error when user has not attached to a channel with state modes - `state_subscribe` mode is required for access/subscribe API - `state_publish` mode is required for create/edit API Resolves DTP-948 --- src/plugins/liveobjects/batchcontext.ts | 1 + .../liveobjects/batchcontextlivecounter.ts | 3 + .../liveobjects/batchcontextlivemap.ts | 4 + src/plugins/liveobjects/livecounter.ts | 3 + src/plugins/liveobjects/livemap.ts | 6 + src/plugins/liveobjects/liveobject.ts | 5 + src/plugins/liveobjects/liveobjects.ts | 33 ++ test/common/modules/private_api_recorder.js | 1 + test/realtime/live_objects.test.js | 295 +++++++++++++----- 9 files changed, 279 insertions(+), 72 deletions(-) diff --git a/src/plugins/liveobjects/batchcontext.ts b/src/plugins/liveobjects/batchcontext.ts index db0dcba34f..e96378248a 100644 --- a/src/plugins/liveobjects/batchcontext.ts +++ b/src/plugins/liveobjects/batchcontext.ts @@ -24,6 +24,7 @@ export class BatchContext { } getRoot(): BatchContextLiveMap { + this._liveObjects.throwIfMissingStateSubscribeMode(); this.throwIfClosed(); return this.getWrappedObject(ROOT_OBJECT_ID) as BatchContextLiveMap; } diff --git a/src/plugins/liveobjects/batchcontextlivecounter.ts b/src/plugins/liveobjects/batchcontextlivecounter.ts index 5462fada2f..e61cd9d1c3 100644 --- a/src/plugins/liveobjects/batchcontextlivecounter.ts +++ b/src/plugins/liveobjects/batchcontextlivecounter.ts @@ -15,17 +15,20 @@ export class BatchContextLiveCounter { } value(): number { + this._liveObjects.throwIfMissingStateSubscribeMode(); this._batchContext.throwIfClosed(); return this._counter.value(); } increment(amount: number): void { + this._liveObjects.throwIfMissingStatePublishMode(); this._batchContext.throwIfClosed(); const stateMessage = LiveCounter.createCounterIncMessage(this._liveObjects, this._counter.getObjectId(), amount); this._batchContext.queueStateMessage(stateMessage); } decrement(amount: number): void { + this._liveObjects.throwIfMissingStatePublishMode(); this._batchContext.throwIfClosed(); // do an explicit type safety check here before negating the amount value, // so we don't unintentionally change the type sent by a user diff --git a/src/plugins/liveobjects/batchcontextlivemap.ts b/src/plugins/liveobjects/batchcontextlivemap.ts index 0c0a84fc7d..9d6fe782dc 100644 --- a/src/plugins/liveobjects/batchcontextlivemap.ts +++ b/src/plugins/liveobjects/batchcontextlivemap.ts @@ -12,6 +12,7 @@ export class BatchContextLiveMap { ) {} get(key: TKey): T[TKey] | undefined { + this._liveObjects.throwIfMissingStateSubscribeMode(); this._batchContext.throwIfClosed(); const value = this._map.get(key); if (value instanceof LiveObject) { @@ -22,17 +23,20 @@ export class BatchContextLiveMap { } size(): number { + this._liveObjects.throwIfMissingStateSubscribeMode(); this._batchContext.throwIfClosed(); return this._map.size(); } set(key: TKey, value: T[TKey]): void { + this._liveObjects.throwIfMissingStatePublishMode(); this._batchContext.throwIfClosed(); const stateMessage = LiveMap.createMapSetMessage(this._liveObjects, this._map.getObjectId(), key, value); this._batchContext.queueStateMessage(stateMessage); } remove(key: TKey): void { + this._liveObjects.throwIfMissingStatePublishMode(); this._batchContext.throwIfClosed(); const stateMessage = LiveMap.createMapRemoveMessage(this._liveObjects, this._map.getObjectId(), key); this._batchContext.queueStateMessage(stateMessage); diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 05425ff626..c5a7be1bbe 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -123,6 +123,7 @@ export class LiveCounter extends LiveObject } value(): number { + this._liveObjects.throwIfMissingStateSubscribeMode(); return this._dataRef.data; } @@ -136,6 +137,7 @@ export class LiveCounter extends LiveObject * @returns A promise which resolves upon receiving the ACK message for the published operation message. */ async increment(amount: number): Promise { + this._liveObjects.throwIfMissingStatePublishMode(); const stateMessage = LiveCounter.createCounterIncMessage(this._liveObjects, this.getObjectId(), amount); return this._liveObjects.publish([stateMessage]); } @@ -144,6 +146,7 @@ export class LiveCounter extends LiveObject * Alias for calling {@link LiveCounter.increment | LiveCounter.increment(-amount)} */ async decrement(amount: number): Promise { + this._liveObjects.throwIfMissingStatePublishMode(); // do an explicit type safety check here before negating the amount value, // so we don't unintentionally change the type sent by a user if (typeof amount !== 'number' || !isFinite(amount)) { diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index f983c52e31..6b9f60e80e 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -266,6 +266,8 @@ export class LiveMap extends LiveObject(key: TKey): T[TKey] | undefined { + this._liveObjects.throwIfMissingStateSubscribeMode(); + if (this.isTombstoned()) { return undefined as T[TKey]; } @@ -303,6 +305,8 @@ export class LiveMap extends LiveObject extends LiveObject(key: TKey, value: T[TKey]): Promise { + this._liveObjects.throwIfMissingStatePublishMode(); const stateMessage = LiveMap.createMapSetMessage(this._liveObjects, this.getObjectId(), key, value); return this._liveObjects.publish([stateMessage]); } @@ -351,6 +356,7 @@ export class LiveMap extends LiveObject(key: TKey): Promise { + this._liveObjects.throwIfMissingStatePublishMode(); const stateMessage = LiveMap.createMapRemoveMessage(this._liveObjects, this.getObjectId(), key); return this._liveObjects.publish([stateMessage]); } diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index 2fe39596fa..084d0dd25c 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -65,6 +65,8 @@ export abstract class LiveObject< } subscribe(listener: (update: TUpdate) => void): SubscribeResponse { + this._liveObjects.throwIfMissingStateSubscribeMode(); + this._eventEmitter.on(LiveObjectEvents.Updated, listener); const unsubscribe = () => { @@ -75,6 +77,8 @@ export abstract class LiveObject< } unsubscribe(listener: (update: TUpdate) => void): void { + // can allow calling this public method without checking for state modes on the channel as the result of this method is not dependant on them + // current implementation of the EventEmitter will remove all listeners if .off is called without arguments or with nullish arguments. // or when called with just an event argument, it will remove all listeners for the event. // thus we need to check that listener does actually exist before calling .off. @@ -86,6 +90,7 @@ export abstract class LiveObject< } unsubscribeAll(): void { + // can allow calling this public method without checking for state modes on the channel as the result of this method is not dependant on them this._eventEmitter.off(LiveObjectEvents.Updated); } diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 1ae68e1cba..3d6286147c 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -49,6 +49,8 @@ export class LiveObjects { * This is useful when working with LiveObjects on multiple channels with different underlying data. */ async getRoot(): Promise> { + this.throwIfMissingStateSubscribeMode(); + // SYNC is currently in progress, wait for SYNC sequence to finish if (this._syncInProgress) { await this._eventEmitter.once(LiveObjectsEvents.SyncCompleted); @@ -61,6 +63,8 @@ export class LiveObjects { * Provides access to the synchronous write API for LiveObjects that can be used to batch multiple operations together in a single channel message. */ async batch(callback: BatchCallback): Promise { + this.throwIfMissingStatePublishMode(); + const root = await this.getRoot(); const context = new BatchContext(this, root); @@ -82,6 +86,8 @@ export class LiveObjects { * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with a zero-value object created in the local pool. */ async createMap(entries?: T): Promise> { + this.throwIfMissingStatePublishMode(); + const stateMessage = await LiveMap.createMapCreateMessage(this, entries); const objectId = stateMessage.operation?.objectId!; @@ -113,6 +119,8 @@ export class LiveObjects { * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with a zero-value object created in the local pool. */ async createCounter(count?: number): Promise { + this.throwIfMissingStatePublishMode(); + const stateMessage = await LiveCounter.createCounterCreateMessage(this, count); const objectId = stateMessage.operation?.objectId!; @@ -246,6 +254,20 @@ export class LiveObjects { return this._channel.sendState(stateMessages); } + /** + * @internal + */ + throwIfMissingStateSubscribeMode(): void { + this._throwIfMissingChannelMode('state_subscribe'); + } + + /** + * @internal + */ + throwIfMissingStatePublishMode(): void { + this._throwIfMissingChannelMode('state_publish'); + } + private _startNewSync(syncId?: string, syncCursor?: string): void { // need to discard all buffered state operation messages on new sync start this._bufferedStateOperations = []; @@ -374,4 +396,15 @@ export class LiveObjects { } } } + + private _throwIfMissingChannelMode(expectedMode: 'state_subscribe' | 'state_publish'): void { + // channel.modes is only populated on channel attachment, so use it only if it is set, + // otherwise as a best effort use user provided channel options + if (this._channel.modes != null && !this._channel.modes.includes(expectedMode)) { + throw new this._client.ErrorInfo(`"${expectedMode}" channel mode must be set for this operation`, 40160, 400); + } + if (!this._client.Utils.allToLowerCase(this._channel.channelOptions.modes ?? []).includes(expectedMode)) { + throw new this._client.ErrorInfo(`"${expectedMode}" channel mode must be set for this operation`, 40160, 400); + } + } } diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index 13ce8f9c94..2898aed40b 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -141,6 +141,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'write.auth.key', 'write.auth.tokenDetails.token', 'write.channel._lastPayload', + 'write.channel.channelOptions.modes', 'write.channel.state', 'write.connectionManager.connectionDetails.maxMessageSize', 'write.connectionManager.connectionId', diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index db76ea78cb..d7bc2cce25 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -57,7 +57,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], return `${paddedTimestamp}-${paddedCounter}@${seriesId}` + (paddedIndex ? `:${paddedIndex}` : ''); } - async function expectRejectedWith(fn, errorStr) { + async function expectToThrowAsync(fn, errorStr) { let verifiedError = false; try { await fn(); @@ -2123,51 +2123,51 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const counter = root.get('counter'); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment(), 'Counter value increment should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment(null), 'Counter value increment should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment(Number.NaN), 'Counter value increment should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment(Number.POSITIVE_INFINITY), 'Counter value increment should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment(Number.NEGATIVE_INFINITY), 'Counter value increment should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment('foo'), 'Counter value increment should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment(BigInt(1)), 'Counter value increment should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment(true), 'Counter value increment should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment(Symbol()), 'Counter value increment should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment({}), 'Counter value increment should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment([]), 'Counter value increment should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.increment(counter), 'Counter value increment should be a valid number', ); @@ -2225,51 +2225,51 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const counter = root.get('counter'); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement(), 'Counter value decrement should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement(null), 'Counter value decrement should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement(Number.NaN), 'Counter value decrement should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement(Number.POSITIVE_INFINITY), 'Counter value decrement should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement(Number.NEGATIVE_INFINITY), 'Counter value decrement should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement('foo'), 'Counter value decrement should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement(BigInt(1)), 'Counter value decrement should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement(true), 'Counter value decrement should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement(Symbol()), 'Counter value decrement should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement({}), 'Counter value decrement should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement([]), 'Counter value decrement should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => counter.decrement(counter), 'Counter value decrement should be a valid number', ); @@ -2351,22 +2351,22 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const map = root.get('map'); - await expectRejectedWith(async () => map.set(), 'Map key should be string'); - await expectRejectedWith(async () => map.set(null), 'Map key should be string'); - await expectRejectedWith(async () => map.set(1), 'Map key should be string'); - await expectRejectedWith(async () => map.set(BigInt(1)), 'Map key should be string'); - await expectRejectedWith(async () => map.set(true), 'Map key should be string'); - await expectRejectedWith(async () => map.set(Symbol()), 'Map key should be string'); - await expectRejectedWith(async () => map.set({}), 'Map key should be string'); - await expectRejectedWith(async () => map.set([]), 'Map key should be string'); - await expectRejectedWith(async () => map.set(map), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(null), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(1), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(BigInt(1)), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(true), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(Symbol()), 'Map key should be string'); + await expectToThrowAsync(async () => map.set({}), 'Map key should be string'); + await expectToThrowAsync(async () => map.set([]), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(map), 'Map key should be string'); - await expectRejectedWith(async () => map.set('key'), 'Map value data type is unsupported'); - await expectRejectedWith(async () => map.set('key', null), 'Map value data type is unsupported'); - await expectRejectedWith(async () => map.set('key', BigInt(1)), 'Map value data type is unsupported'); - await expectRejectedWith(async () => map.set('key', Symbol()), 'Map value data type is unsupported'); - await expectRejectedWith(async () => map.set('key', {}), 'Map value data type is unsupported'); - await expectRejectedWith(async () => map.set('key', []), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => map.set('key'), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => map.set('key', null), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => map.set('key', BigInt(1)), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => map.set('key', Symbol()), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => map.set('key', {}), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => map.set('key', []), 'Map value data type is unsupported'); }, }, @@ -2414,15 +2414,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const map = root.get('map'); - await expectRejectedWith(async () => map.remove(), 'Map key should be string'); - await expectRejectedWith(async () => map.remove(null), 'Map key should be string'); - await expectRejectedWith(async () => map.remove(1), 'Map key should be string'); - await expectRejectedWith(async () => map.remove(BigInt(1)), 'Map key should be string'); - await expectRejectedWith(async () => map.remove(true), 'Map key should be string'); - await expectRejectedWith(async () => map.remove(Symbol()), 'Map key should be string'); - await expectRejectedWith(async () => map.remove({}), 'Map key should be string'); - await expectRejectedWith(async () => map.remove([]), 'Map key should be string'); - await expectRejectedWith(async () => map.remove(map), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(null), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(1), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(BigInt(1)), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(true), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(Symbol()), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove({}), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove([]), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(map), 'Map key should be string'); }, }, @@ -2552,47 +2552,47 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjects } = ctx; - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createCounter(null), 'Counter value should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createCounter(Number.NaN), 'Counter value should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createCounter(Number.POSITIVE_INFINITY), 'Counter value should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createCounter(Number.NEGATIVE_INFINITY), 'Counter value should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createCounter('foo'), 'Counter value should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createCounter(BigInt(1)), 'Counter value should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createCounter(true), 'Counter value should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createCounter(Symbol()), 'Counter value should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createCounter({}), 'Counter value should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createCounter([]), 'Counter value should be a valid number', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createCounter(root), 'Counter value should be a valid number', ); @@ -2805,49 +2805,49 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjects } = ctx; - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createMap(null), 'Map entries should be a key/value object', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createMap('foo'), 'Map entries should be a key/value object', ); - await expectRejectedWith(async () => liveObjects.createMap(1), 'Map entries should be a key/value object'); - await expectRejectedWith( + await expectToThrowAsync(async () => liveObjects.createMap(1), 'Map entries should be a key/value object'); + await expectToThrowAsync( async () => liveObjects.createMap(BigInt(1)), 'Map entries should be a key/value object', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createMap(true), 'Map entries should be a key/value object', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createMap(Symbol()), 'Map entries should be a key/value object', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createMap({ key: undefined }), 'Map value data type is unsupported', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createMap({ key: null }), 'Map value data type is unsupported', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createMap({ key: BigInt(1) }), 'Map value data type is unsupported', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createMap({ key: Symbol() }), 'Map value data type is unsupported', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createMap({ key: {} }), 'Map value data type is unsupported', ); - await expectRejectedWith( + await expectToThrowAsync( async () => liveObjects.createMap({ key: [] }), 'Map value data type is unsupported', ); @@ -3814,6 +3814,157 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], LiveObjectsPlugin.LiveObjects._DEFAULTS.gcGracePeriod = gcGracePeriodOriginal; } }); + + const expectToThrowMissingStateMode = async ({ liveObjects, map, counter }) => { + await expectToThrowAsync( + async () => liveObjects.getRoot(), + '"state_subscribe" channel mode must be set for this operation', + ); + await expectToThrowAsync( + async () => liveObjects.batch(), + '"state_publish" channel mode must be set for this operation', + ); + await expectToThrowAsync( + async () => liveObjects.createMap(), + '"state_publish" channel mode must be set for this operation', + ); + await expectToThrowAsync( + async () => liveObjects.createCounter(), + '"state_publish" channel mode must be set for this operation', + ); + + expect(() => counter.value()).to.throw('"state_subscribe" channel mode must be set for this operation'); + await expectToThrowAsync( + async () => counter.increment(), + '"state_publish" channel mode must be set for this operation', + ); + await expectToThrowAsync( + async () => counter.decrement(), + '"state_publish" channel mode must be set for this operation', + ); + + expect(() => map.get()).to.throw('"state_subscribe" channel mode must be set for this operation'); + expect(() => map.size()).to.throw('"state_subscribe" channel mode must be set for this operation'); + await expectToThrowAsync(async () => map.set(), '"state_publish" channel mode must be set for this operation'); + await expectToThrowAsync( + async () => map.remove(), + '"state_publish" channel mode must be set for this operation', + ); + + for (const obj of [map, counter]) { + expect(() => obj.subscribe()).to.throw('"state_subscribe" channel mode must be set for this operation'); + expect(() => obj.unsubscribe(() => {})).not.to.throw( + '"state_subscribe" channel mode must be set for this operation', + ); // note: this should not throw + expect(() => obj.unsubscribe(() => {})).not.to.throw( + '"state_publish" channel mode must be set for this operation', + ); // note: this should not throw + expect(() => obj.unsubscribeAll()).not.to.throw( + '"state_subscribe" channel mode must be set for this operation', + ); // note: this should not throw + expect(() => obj.unsubscribeAll()).not.to.throw( + '"state_publish" channel mode must be set for this operation', + ); // note: this should not throw + } + }; + + const expectToThrowMissingStateModeInBatchContext = ({ ctx, map, counter }) => { + expect(() => ctx.getRoot()).to.throw('"state_subscribe" channel mode must be set for this operation'); + + expect(() => counter.value()).to.throw('"state_subscribe" channel mode must be set for this operation'); + expect(() => counter.increment()).to.throw('"state_publish" channel mode must be set for this operation'); + expect(() => counter.decrement()).to.throw('"state_publish" channel mode must be set for this operation'); + + expect(() => map.get()).to.throw('"state_subscribe" channel mode must be set for this operation'); + expect(() => map.size()).to.throw('"state_subscribe" channel mode must be set for this operation'); + expect(() => map.set()).to.throw('"state_publish" channel mode must be set for this operation'); + expect(() => map.remove()).to.throw('"state_publish" channel mode must be set for this operation'); + }; + + const missingChannelModesScenarios = [ + { + description: 'public API throws missing state modes error when attached without correct state modes', + action: async (ctx) => { + const { liveObjects, channel, map, counter } = ctx; + + // obtain batch context with valid modes first + await liveObjects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate missing modes + channel.modes = []; + expectToThrowMissingStateModeInBatchContext({ ctx, map, counter }); + }); + await expectToThrowMissingStateMode({ liveObjects, map, counter }); + }, + }, + + { + description: + 'public API throws missing state modes error when not yet attached but client options are missing correct modes', + action: async (ctx) => { + const { liveObjects, channel, map, counter, helper } = ctx; + + // obtain batch context with valid modes first + await liveObjects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate a situation where we're not yet attached/modes are not received on ATTACHED event + channel.modes = undefined; + helper.recordPrivateApi('write.channel.channelOptions.modes'); + channel.channelOptions.modes = []; + + expectToThrowMissingStateModeInBatchContext({ ctx, map, counter }); + }); + await expectToThrowMissingStateMode({ liveObjects, map, counter }); + }, + }, + ]; + + /** @nospec */ + forScenarios(missingChannelModesScenarios, async function (helper, scenario) { + const liveObjectsHelper = new LiveObjectsHelper(helper); + const client = RealtimeWithLiveObjects(helper); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + const channelName = scenario.description; + // attach with correct channel modes so we can create liveobjects on the root for testing. + // each scenario will modify the underlying modes array to test specific behavior + const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + let mapSet = false; + let counterSet = false; + const rootReadyPromise = new Promise((resolve) => { + const { unsubscribe } = root.subscribe(({ update }) => { + if (update.map) { + mapSet = true; + } + if (update.counter) { + counterSet = true; + } + + if (mapSet && counterSet) { + unsubscribe(); + resolve(); + } + }); + }); + + const map = await liveObjects.createMap(); + const counter = await liveObjects.createCounter(); + + await root.set('map', map); + await root.set('counter', counter); + + await rootReadyPromise; + + await scenario.action({ liveObjects, liveObjectsHelper, channelName, channel, root, map, counter, helper }); + }, client); + }); }); /** @nospec */ From 702420f34eb6655e49a0286f3da98aec99696877 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 29 Jan 2025 01:18:38 +0000 Subject: [PATCH 112/166] Fix fake object id functions in live objects helper --- src/plugins/liveobjects/objectid.ts | 1 - test/common/modules/live_objects_helper.js | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/liveobjects/objectid.ts b/src/plugins/liveobjects/objectid.ts index 2e5a817f75..35bd811485 100644 --- a/src/plugins/liveobjects/objectid.ts +++ b/src/plugins/liveobjects/objectid.ts @@ -29,7 +29,6 @@ export class ObjectId { platform.BufferUtils.utf8Encode(nonce), ]); const hashBuffer = platform.BufferUtils.sha256(valueForHashBuffer); - const hash = platform.BufferUtils.base64UrlEncode(hashBuffer); return new ObjectId(objectType, hash, msTimestamp); diff --git a/test/common/modules/live_objects_helper.js b/test/common/modules/live_objects_helper.js index b5e42f79de..df0db156fd 100644 --- a/test/common/modules/live_objects_helper.js +++ b/test/common/modules/live_objects_helper.js @@ -292,11 +292,11 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb } fakeMapObjectId() { - return `map:${Helper.randomString()}`; + return `map:${Helper.randomString()}@${Date.now()}`; } fakeCounterObjectId() { - return `counter:${Helper.randomString()}`; + return `counter:${Helper.randomString()}@${Date.now()}`; } async stateRequest(channelName, opBody) { From 9ef65e65397dccaa8914fb4b123f5c9bc131d155 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 29 Jan 2025 04:15:28 +0000 Subject: [PATCH 113/166] Fix `primitiveKeyData` sample data for was modified by some live objects tests --- test/realtime/live_objects.test.js | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index d7bc2cce25..9d71a0c283 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -1769,7 +1769,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], + // copy data object as library will modify it + state: [ + liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } }), + ], }), ), ); @@ -1800,7 +1803,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', - state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], + // copy data object as library will modify it + state: [ + liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } }), + ], }), ), ); @@ -1846,7 +1852,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', - state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], + // copy data object as library will modify it + state: [ + liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } }), + ], }), ), ); @@ -2028,7 +2037,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', - state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data })], + // copy data object as library will modify it + state: [ + liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } }), + ], }), ), ); From dc90742a070539022621c9c7b25e59b1b6ffe7d3 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 29 Jan 2025 03:59:37 +0000 Subject: [PATCH 114/166] Fix flaky live objects tests that relied on STATE/STATE_SYNC messages that client may not have received Resolves DTP-1147 --- test/common/modules/live_objects_helper.js | 8 +- test/realtime/live_objects.test.js | 257 ++++++++++++++++++--- 2 files changed, 237 insertions(+), 28 deletions(-) diff --git a/test/common/modules/live_objects_helper.js b/test/common/modules/live_objects_helper.js index df0db156fd..aa35e77397 100644 --- a/test/common/modules/live_objects_helper.js +++ b/test/common/modules/live_objects_helper.js @@ -25,8 +25,14 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb this._rest = helper.AblyRest({ useBinaryProtocol: false }); } + static ACTIONS = ACTIONS; + + static fixtureRootKeys() { + return ['emptyCounter', 'initialValueCounter', 'referencedCounter', 'emptyMap', 'referencedMap', 'valuesMap']; + } + /** - * Creates next LiveObjects state tree on a provided channel name: + * Sends REST STATE requests to create LiveObjects state tree on a provided channel name: * * root "emptyMap" -> Map#1 {} -- empty map * root "referencedMap" -> Map#2 { "counterKey": } diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 9d71a0c283..74eff8a33b 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -68,6 +68,65 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(verifiedError, 'Expected async function to throw an error').to.be.true; } + async function waitForMapKeyUpdate(map, key) { + return new Promise((resolve) => { + const { unsubscribe } = map.subscribe(({ update }) => { + if (update[key]) { + unsubscribe(); + resolve(); + } + }); + }); + } + + async function waitForCounterUpdate(counter) { + return new Promise((resolve) => { + const { unsubscribe } = counter.subscribe(() => { + unsubscribe(); + resolve(); + }); + }); + } + + async function waitForStateOperation(helper, client, waitForAction) { + return new Promise((resolve, reject) => { + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + const transport = client.connection.connectionManager.activeProtocol.getTransport(); + const onProtocolMessageOriginal = transport.onProtocolMessage; + + helper.recordPrivateApi('replace.transport.onProtocolMessage'); + transport.onProtocolMessage = function (message) { + try { + helper.recordPrivateApi('call.transport.onProtocolMessage'); + onProtocolMessageOriginal.call(transport, message); + + if (message.action === 19 && message.state[0]?.operation?.action === waitForAction) { + helper.recordPrivateApi('replace.transport.onProtocolMessage'); + transport.onProtocolMessage = onProtocolMessageOriginal; + resolve(); + } + } catch (err) { + reject(err); + } + }; + }); + } + + /** + * The channel with fixture data may not yet be populated by REST STATE requests made by LiveObjectsHelper. + * This function waits for a channel to have all keys set. + */ + async function waitFixtureChannelIsReady(client) { + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + const expectedKeys = LiveObjectsHelper.fixtureRootKeys(); + + await channel.attach(); + const root = await liveObjects.getRoot(); + + await Promise.all(expectedKeys.map((key) => (root.get(key) ? undefined : waitForMapKeyUpdate(root, key)))); + } + describe('realtime/live_objects', function () { this.timeout(60 * 1000); @@ -342,6 +401,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const client = RealtimeWithLiveObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { + await waitFixtureChannelIsReady(client); + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); const liveObjects = channel.liveObjects; @@ -393,6 +454,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const client = RealtimeWithLiveObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { + await waitFixtureChannelIsReady(client); + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); const liveObjects = channel.liveObjects; @@ -418,6 +481,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const client = RealtimeWithLiveObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { + await waitFixtureChannelIsReady(client); + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); const liveObjects = channel.liveObjects; @@ -465,6 +530,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const client = RealtimeWithLiveObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { + await waitFixtureChannelIsReady(client); + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); const liveObjects = channel.liveObjects; @@ -588,11 +655,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName, channel } = ctx; + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); const { objectId: counterId } = await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', createOp: liveObjectsHelper.counterCreateOp({ count: 1 }), }); + await counterCreatedPromise; expect(root.get('counter'), 'Check counter exists on root before STATE_SYNC sequence with "tombstone=true"') .to.exist; @@ -636,11 +705,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName, channel } = ctx; + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); const { objectId: counterId } = await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', createOp: liveObjectsHelper.counterCreateOp({ count: 1 }), }); + await counterCreatedPromise; const counterSubPromise = new Promise((resolve, reject) => root.get('counter').subscribe((update) => { @@ -701,6 +772,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], .exist; }); + const mapsCreatedPromise = Promise.all(primitiveMapsFixtures.map((x) => waitForMapKeyUpdate(root, x.name))); // create new maps and set on root await Promise.all( primitiveMapsFixtures.map((fixture) => @@ -711,6 +783,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }), ), ); + await mapsCreatedPromise; // check created maps primitiveMapsFixtures.forEach((fixture) => { @@ -760,6 +833,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], `Check "${withReferencesMapKey}" key doesn't exist on root before applying MAP_CREATE ops`, ).to.not.exist; + const mapCreatedPromise = waitForMapKeyUpdate(root, withReferencesMapKey); // create map with references. need to create referenced objects first to obtain their object ids const { objectId: referencedMapObjectId } = await liveObjectsHelper.stateRequest( channelName, @@ -779,6 +853,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, }), }); + await mapCreatedPromise; // check map with references exist on root const withReferencesMap = root.get(withReferencesMapKey); @@ -912,6 +987,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ).to.not.exist; }); + const keysUpdatedPromise = Promise.all(primitiveKeyData.map((x) => waitForMapKeyUpdate(root, x.key))); // apply MAP_SET ops await Promise.all( primitiveKeyData.map((keyData) => @@ -925,6 +1001,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ), ), ); + await keysUpdatedPromise; // check everything is applied correctly primitiveKeyData.forEach((keyData) => { @@ -956,6 +1033,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(root.get('keyToMap'), `Check "keyToMap" key doesn't exist on root before applying MAP_SET ops`).to .not.exist; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'keyToCounter'), + waitForMapKeyUpdate(root, 'keyToMap'), + ]); // create new objects and set on root await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', @@ -972,6 +1053,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, }), }); + await objectsCreatedPromise; // check root has refs to new objects and they are not zero-value const counter = root.get('keyToCounter'); @@ -1070,6 +1152,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const { root, liveObjectsHelper, channelName } = ctx; const mapKey = 'map'; + const mapCreatedPromise = waitForMapKeyUpdate(root, mapKey); // create new map and set on root const { objectId: mapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', @@ -1081,6 +1164,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, }), }); + await mapCreatedPromise; const map = root.get(mapKey); // check map has expected keys before MAP_REMOVE ops @@ -1097,6 +1181,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], `Check map at "${mapKey}" key in root has correct "shouldDelete" value before MAP_REMOVE`, ); + const keyRemovedPromise = waitForMapKeyUpdate(map, 'shouldDelete'); // send MAP_REMOVE op await liveObjectsHelper.stateRequest( channelName, @@ -1105,6 +1190,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], key: 'shouldDelete', }), ); + await keyRemovedPromise; // check map has correct keys after MAP_REMOVE ops expect(map.size()).to.equal( @@ -1210,6 +1296,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], .not.exist; }); + const countersCreatedPromise = Promise.all(countersFixtures.map((x) => waitForMapKeyUpdate(root, x.name))); // create new counters and set on root await Promise.all( countersFixtures.map((fixture) => @@ -1220,6 +1307,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }), ), ); + await countersCreatedPromise; // check created counters countersFixtures.forEach((fixture) => { @@ -1321,12 +1409,14 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const counterKey = 'counter'; let expectedCounterValue = 0; + const counterCreated = waitForMapKeyUpdate(root, counterKey); // create new counter and set on root const { objectId: counterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: counterKey, createOp: liveObjectsHelper.counterCreateOp({ count: expectedCounterValue }), }); + await counterCreated; const counter = root.get(counterKey); // check counter has expected value before COUNTER_INC @@ -1355,7 +1445,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // send increments one at a time and check expected value for (let i = 0; i < increments.length; i++) { const increment = increments[i]; + expectedCounterValue += increment; + const counterUpdatedPromise = waitForCounterUpdate(counter); await liveObjectsHelper.stateRequest( channelName, liveObjectsHelper.counterIncOp({ @@ -1363,7 +1455,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], amount: increment, }), ); - expectedCounterValue += increment; + await counterUpdatedPromise; expect(counter.value()).to.equal( expectedCounterValue, @@ -1424,6 +1516,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName, channel } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'map'), + waitForMapKeyUpdate(root, 'counter'), + ]); // create initial objects and set on root const { objectId: mapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', @@ -1435,6 +1531,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], key: 'counter', createOp: liveObjectsHelper.counterCreateOp(), }); + await objectsCreatedPromise; expect(root.get('map'), 'Check map exists on root before OBJECT_DELETE').to.exist; expect(root.get('counter'), 'Check counter exists on root before OBJECT_DELETE').to.exist; @@ -1572,6 +1669,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName, channel } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'map'), + waitForMapKeyUpdate(root, 'counter'), + ]); // create initial objects and set on root const { objectId: mapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', @@ -1588,6 +1689,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], key: 'counter', createOp: liveObjectsHelper.counterCreateOp({ count: 1 }), }); + await objectsCreatedPromise; const mapSubPromise = new Promise((resolve, reject) => root.get('map').subscribe((update) => { @@ -1639,12 +1741,14 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName, channel } = ctx; + const objectCreatedPromise = waitForMapKeyUpdate(root, 'foo'); // create initial objects and set on root const { objectId: counterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'foo', createOp: liveObjectsHelper.counterCreateOp(), }); + await objectCreatedPromise; expect(root.get('foo'), 'Check counter exists on root before OBJECT_DELETE').to.exist; @@ -1676,6 +1780,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName, channel } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'map1'), + waitForMapKeyUpdate(root, 'map2'), + waitForMapKeyUpdate(root, 'counter1'), + ]); // create initial objects and set on root const { objectId: mapId1 } = await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', @@ -1692,6 +1801,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], key: 'counter1', createOp: liveObjectsHelper.counterCreateOp(), }); + await objectsCreatedPromise; expect(root.get('map1'), 'Check map1 exists on root before OBJECT_DELETE').to.exist; expect(root.get('map2'), 'Check map2 exists on root before OBJECT_DELETE').to.exist; @@ -2051,6 +2161,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], syncSerial: 'serial:', }); + const keyUpdatedPromise = waitForMapKeyUpdate(root, 'foo'); // send some more operations await liveObjectsHelper.stateRequest( channelName, @@ -2060,6 +2171,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], data: { value: 'bar' }, }), ); + await keyUpdatedPromise; // check buffered operations are applied, as well as the most recent operation outside of the STATE_SYNC is applied primitiveKeyData.forEach((keyData) => { @@ -2089,11 +2201,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', createOp: liveObjectsHelper.counterCreateOp(), }); + await counterCreatedPromise; const counter = root.get('counter'); const increments = [ @@ -2112,7 +2226,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], for (let i = 0; i < increments.length; i++) { const increment = increments[i]; expectedCounterValue += increment; + + const counterUpdatedPromise = waitForCounterUpdate(counter); await counter.increment(increment); + await counterUpdatedPromise; expect(counter.value()).to.equal( expectedCounterValue, @@ -2127,11 +2244,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', createOp: liveObjectsHelper.counterCreateOp(), }); + await counterCreatedPromise; const counter = root.get('counter'); @@ -2191,11 +2310,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', createOp: liveObjectsHelper.counterCreateOp(), }); + await counterCreatedPromise; const counter = root.get('counter'); const decrements = [ @@ -2214,7 +2335,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], for (let i = 0; i < decrements.length; i++) { const decrement = decrements[i]; expectedCounterValue -= decrement; + + const counterUpdatedPromise = waitForCounterUpdate(counter); await counter.decrement(decrement); + await counterUpdatedPromise; expect(counter.value()).to.equal( expectedCounterValue, @@ -2229,11 +2353,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', createOp: liveObjectsHelper.counterCreateOp(), }); + await counterCreatedPromise; const counter = root.get('counter'); @@ -2293,12 +2419,14 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root } = ctx; + const keysUpdatedPromise = Promise.all(primitiveKeyData.map((x) => waitForMapKeyUpdate(root, x.key))); await Promise.all( primitiveKeyData.map(async (keyData) => { const value = keyData.data.encoding ? BufferUtils.base64Decode(keyData.data.value) : keyData.data.value; await root.set(keyData.key, value); }), ); + await keysUpdatedPromise; // check everything is applied correctly primitiveKeyData.forEach((keyData) => { @@ -2322,6 +2450,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', @@ -2332,12 +2464,18 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], key: 'map', createOp: liveObjectsHelper.mapCreateOp(), }); + await objectsCreatedPromise; const counter = root.get('counter'); const map = root.get('map'); + const keysUpdatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter2'), + waitForMapKeyUpdate(root, 'map2'), + ]); await root.set('counter2', counter); await root.set('map2', map); + await keysUpdatedPromise; expect(root.get('counter2')).to.equal( counter, @@ -2355,11 +2493,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; + const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', createOp: liveObjectsHelper.mapCreateOp(), }); + await mapCreatedPromise; const map = root.get('map'); @@ -2387,6 +2527,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; + const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', @@ -2398,11 +2539,14 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, }), }); + await mapCreatedPromise; const map = root.get('map'); + const keysUpdatedPromise = Promise.all([waitForMapKeyUpdate(map, 'foo'), waitForMapKeyUpdate(map, 'bar')]); await map.remove('foo'); await map.remove('bar'); + await keysUpdatedPromise; expect(map.get('foo'), 'Check can remove a key from a root via a LiveMap.remove call').to.not.exist; expect(map.get('bar'), 'Check can remove a key from a root via a LiveMap.remove call').to.not.exist; @@ -2418,11 +2562,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; + const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', createOp: liveObjectsHelper.mapCreateOp(), }); + await mapCreatedPromise; const map = root.get('map'); @@ -2464,8 +2610,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjects } = ctx; + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); const counter = await liveObjects.createCounter(1); await root.set('counter', counter); + await counterCreatedPromise; expectInstanceOf(counter, 'LiveCounter', `Check counter instance is of an expected class`); expectInstanceOf( @@ -2510,7 +2658,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], helper.recordPrivateApi('replace.LiveObjects.publish'); liveObjects.publish = async (stateMessages) => { const counterId = stateMessages[0].operation.objectId; - // this should result in liveobjects' operation application procedure and create a object in the pool with forged initial value + // this should result in liveobjects' operation application procedure and create an object in the pool with forged initial value await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 1), @@ -2666,6 +2814,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName, liveObjects } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', @@ -2676,6 +2828,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], key: 'map', createOp: liveObjectsHelper.mapCreateOp(), }); + await objectsCreatedPromise; const counter = root.get('counter'); const map = root.get('map'); @@ -2701,9 +2854,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjects } = ctx; + const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); const counter = await liveObjects.createCounter(); const map = await liveObjects.createMap({ foo: 'bar', baz: counter }); await root.set('map', map); + await mapCreatedPromise; expectInstanceOf(map, 'LiveMap', `Check map instance is of an expected class`); expectInstanceOf(root.get('map'), 'LiveMap', `Check map instance on root is of an expected class`); @@ -2747,7 +2902,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], helper.recordPrivateApi('replace.LiveObjects.publish'); liveObjects.publish = async (stateMessages) => { const mapId = stateMessages[0].operation.objectId; - // this should result in liveobjects' operation application procedure and create a object in the pool with forged initial value + // this should result in liveobjects' operation application procedure and create an object in the pool with forged initial value await liveObjectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 1), @@ -2884,10 +3039,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjects } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); const counter = await liveObjects.createCounter(1); const map = await liveObjects.createMap({ innerCounter: counter }); await root.set('counter', counter); await root.set('map', map); + await objectsCreatedPromise; await liveObjects.batch((ctx) => { const ctxRoot = ctx.getRoot(); @@ -2922,10 +3082,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjects } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); const counter = await liveObjects.createCounter(1); const map = await liveObjects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); + await objectsCreatedPromise; await liveObjects.batch((ctx) => { const ctxRoot = ctx.getRoot(); @@ -2947,10 +3112,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjects } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); const counter = await liveObjects.createCounter(1); const map = await liveObjects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); + await objectsCreatedPromise; await liveObjects.batch((ctx) => { const ctxRoot = ctx.getRoot(); @@ -2989,10 +3159,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjects } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); const counter = await liveObjects.createCounter(1); const map = await liveObjects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); + await objectsCreatedPromise; await liveObjects.batch((ctx) => { const ctxRoot = ctx.getRoot(); @@ -3035,10 +3210,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjects } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); const counter = await liveObjects.createCounter(1); const map = await liveObjects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); + await objectsCreatedPromise; const cancelError = new Error('cancel batch'); let caughtError; @@ -3075,10 +3255,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjects } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); const counter = await liveObjects.createCounter(1); const map = await liveObjects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); + await objectsCreatedPromise; let savedCtx; let savedCtxCounter; @@ -3103,14 +3288,19 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { - description: `batch API batch context and derived objects can't be interacted with thrown error from batch callback`, + description: `batch API batch context and derived objects can't be interacted with after error was thrown from batch callback`, action: async (ctx) => { const { root, liveObjects } = ctx; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); const counter = await liveObjects.createCounter(1); const map = await liveObjects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); + await objectsCreatedPromise; let savedCtx; let savedCtxCounter; @@ -3408,6 +3598,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const increments = 3; for (let i = 0; i < increments; i++) { + const counterUpdatedPromise = waitForCounterUpdate(counter); await liveObjectsHelper.stateRequest( channelName, liveObjectsHelper.counterIncOp({ @@ -3415,6 +3606,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], amount: 1, }), ); + await counterUpdatedPromise; } await subscriptionPromise; @@ -3444,6 +3636,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const increments = 3; for (let i = 0; i < increments; i++) { + const counterUpdatedPromise = waitForCounterUpdate(counter); await liveObjectsHelper.stateRequest( channelName, liveObjectsHelper.counterIncOp({ @@ -3451,6 +3644,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], amount: 1, }), ); + await counterUpdatedPromise; } await subscriptionPromise; @@ -3482,6 +3676,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const increments = 3; for (let i = 0; i < increments; i++) { + const counterUpdatedPromise = waitForCounterUpdate(counter); await liveObjectsHelper.stateRequest( channelName, liveObjectsHelper.counterIncOp({ @@ -3489,6 +3684,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], amount: 1, }), ); + await counterUpdatedPromise; if (i === 0) { // unsub all after first operation @@ -3521,6 +3717,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const mapSets = 3; for (let i = 0; i < mapSets; i++) { + const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); await liveObjectsHelper.stateRequest( channelName, liveObjectsHelper.mapSetOp({ @@ -3529,6 +3726,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], data: { value: 'exists' }, }), ); + await mapUpdatedPromise; } await subscriptionPromise; @@ -3563,6 +3761,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const mapSets = 3; for (let i = 0; i < mapSets; i++) { + const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); await liveObjectsHelper.stateRequest( channelName, liveObjectsHelper.mapSetOp({ @@ -3571,6 +3770,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], data: { value: 'exists' }, }), ); + await mapUpdatedPromise; } await subscriptionPromise; @@ -3607,6 +3807,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const mapSets = 3; for (let i = 0; i < mapSets; i++) { + const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); await liveObjectsHelper.stateRequest( channelName, liveObjectsHelper.mapSetOp({ @@ -3615,6 +3816,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], data: { value: 'exists' }, }), ); + await mapUpdatedPromise; if (i === 0) { // unsub all after first operation @@ -3651,6 +3853,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const sampleMapKey = 'sampleMap'; const sampleCounterKey = 'sampleCounter'; + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, sampleMapKey), + waitForMapKeyUpdate(root, sampleCounterKey), + ]); // prepare map and counter objects for use by the scenario const { objectId: sampleMapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', @@ -3662,6 +3868,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], key: sampleCounterKey, createOp: liveObjectsHelper.counterCreateOp(), }); + await objectsCreatedPromise; await scenario.action({ root, @@ -3682,13 +3889,19 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'tombstoned object is removed from the pool after the GC grace period', action: async (ctx) => { - const { liveObjectsHelper, channelName, channel, liveObjects, helper, waitForGCCycles } = ctx; + const { liveObjectsHelper, channelName, channel, liveObjects, helper, waitForGCCycles, client } = ctx; - // send a CREATE op, this add an object to the pool + const counterCreatedPromise = waitForStateOperation( + helper, + client, + LiveObjectsHelper.ACTIONS.COUNTER_CREATE, + ); + // send a CREATE op, this adds an object to the pool const { objectId } = await liveObjectsHelper.stateRequest( channelName, liveObjectsHelper.counterCreateOp({ count: 1 }), ); + await counterCreatedPromise; expect(liveObjects._liveObjectsPool.get(objectId), 'Check object exists in the pool after creation').to .exist; @@ -3730,19 +3943,23 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], action: async (ctx) => { const { root, liveObjectsHelper, channelName, helper, waitForGCCycles } = ctx; + const keyUpdatedPromise = waitForMapKeyUpdate(root, 'foo'); // set a key on a root await liveObjectsHelper.stateRequest( channelName, liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { value: 'bar' } }), ); + await keyUpdatedPromise; expect(root.get('foo')).to.equal('bar', 'Check key "foo" exists on root after MAP_SET'); + const keyUpdatedPromise2 = waitForMapKeyUpdate(root, 'foo'); // remove the key from the root. this should tombstone the map entry and make it inaccessible to the end user, but still keep it in memory in the underlying map await liveObjectsHelper.stateRequest( channelName, liveObjectsHelper.mapRemoveOp({ objectId: 'root', key: 'foo' }), ); + await keyUpdatedPromise2; expect(root.get('foo'), 'Check key "foo" is inaccessible via public API on root after MAP_REMOVE').to.not .exist; @@ -3810,6 +4027,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }; await scenario.action({ + client, root, liveObjectsHelper, channelName, @@ -3880,6 +4098,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], } }; + /** Make sure to call this inside the batch method as batch objects can't be interacted with outside the batch callback */ const expectToThrowMissingStateModeInBatchContext = ({ ctx, map, counter }) => { expect(() => ctx.getRoot()).to.throw('"state_subscribe" channel mode must be set for this operation'); @@ -3948,31 +4167,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await channel.attach(); const root = await liveObjects.getRoot(); - let mapSet = false; - let counterSet = false; - const rootReadyPromise = new Promise((resolve) => { - const { unsubscribe } = root.subscribe(({ update }) => { - if (update.map) { - mapSet = true; - } - if (update.counter) { - counterSet = true; - } - - if (mapSet && counterSet) { - unsubscribe(); - resolve(); - } - }); - }); - + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'map'), + waitForMapKeyUpdate(root, 'counter'), + ]); const map = await liveObjects.createMap(); const counter = await liveObjects.createCounter(); - await root.set('map', map); await root.set('counter', counter); - - await rootReadyPromise; + await objectsCreatedPromise; await scenario.action({ liveObjects, liveObjectsHelper, channelName, channel, root, map, counter, helper }); }, client); From 98d66e26f8270ddc422b8b9f016572cc8fefb3d9 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 31 Jan 2025 03:43:41 +0000 Subject: [PATCH 115/166] Refactor `LiveObjects` to maintain internal state sync state --- src/plugins/liveobjects/liveobjects.ts | 48 ++++++++++++++++++++------ 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 3d6286147c..b52cb809cf 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -11,21 +11,34 @@ import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage, StateOperationAction } from './statemessage'; import { SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; -enum LiveObjectsEvents { - SyncCompleted = 'SyncCompleted', +export enum LiveObjectsEvent { + syncing = 'syncing', + synced = 'synced', } +export enum LiveObjectsState { + initialized = 'initialized', + syncing = 'syncing', + synced = 'synced', +} + +const StateToEventsMap: Record = { + initialized: undefined, + syncing: LiveObjectsEvent.syncing, + synced: LiveObjectsEvent.synced, +}; + type BatchCallback = (batchContext: BatchContext) => void; export class LiveObjects { private _client: BaseClient; private _channel: RealtimeChannel; + private _state: LiveObjectsState; // composition over inheritance since we cannot import class directly into plugin code. // instead we obtain a class type from the client private _eventEmitter: EventEmitter; private _liveObjectsPool: LiveObjectsPool; private _syncLiveObjectsDataPool: SyncLiveObjectsDataPool; - private _syncInProgress: boolean; private _currentSyncId: string | undefined; private _currentSyncCursor: string | undefined; private _bufferedStateOperations: StateMessage[]; @@ -36,10 +49,10 @@ export class LiveObjects { constructor(channel: RealtimeChannel) { this._channel = channel; this._client = channel.client; + this._state = LiveObjectsState.initialized; this._eventEmitter = new this._client.EventEmitter(this._client.logger); this._liveObjectsPool = new LiveObjectsPool(this); this._syncLiveObjectsDataPool = new SyncLiveObjectsDataPool(this); - this._syncInProgress = true; this._bufferedStateOperations = []; } @@ -51,9 +64,9 @@ export class LiveObjects { async getRoot(): Promise> { this.throwIfMissingStateSubscribeMode(); - // SYNC is currently in progress, wait for SYNC sequence to finish - if (this._syncInProgress) { - await this._eventEmitter.once(LiveObjectsEvents.SyncCompleted); + // if we're not synced yet, wait for SYNC sequence to finish before returning root + if (this._state !== LiveObjectsState.synced) { + await this._eventEmitter.once(LiveObjectsEvent.synced); } return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; @@ -183,7 +196,7 @@ export class LiveObjects { * @internal */ handleStateMessages(stateMessages: StateMessage[]): void { - if (this._syncInProgress) { + if (this._state !== LiveObjectsState.synced) { // The client receives state messages in realtime over the channel concurrently with the SYNC sequence. // Some of the incoming state messages may have already been applied to the state objects described in // the SYNC sequence, but others may not; therefore we must buffer these messages so that we can apply @@ -274,7 +287,7 @@ export class LiveObjects { this._syncLiveObjectsDataPool.reset(); this._currentSyncId = syncId; this._currentSyncCursor = syncCursor; - this._syncInProgress = true; + this._stateChange(LiveObjectsState.syncing); } private _endSync(): void { @@ -287,8 +300,7 @@ export class LiveObjects { this._syncLiveObjectsDataPool.reset(); this._currentSyncId = undefined; this._currentSyncCursor = undefined; - this._syncInProgress = false; - this._eventEmitter.emit(LiveObjectsEvents.SyncCompleted); + this._stateChange(LiveObjectsState.synced); } private _parseSyncChannelSerial(syncChannelSerial: string | null | undefined): { @@ -407,4 +419,18 @@ export class LiveObjects { throw new this._client.ErrorInfo(`"${expectedMode}" channel mode must be set for this operation`, 40160, 400); } } + + private _stateChange(state: LiveObjectsState): void { + if (this._state === state) { + return; + } + + this._state = state; + const event = StateToEventsMap[state]; + if (!event) { + return; + } + + this._eventEmitter.emit(event); + } } From cf6f5fd5cd44c8d43f7aef6da80683e6659f077e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 23 Jan 2025 04:26:12 +0000 Subject: [PATCH 116/166] Expose state sync events on `LiveObjects` via public API DTP-1034 --- src/plugins/liveobjects/liveobjects.ts | 85 +++++++++++++++++++++----- 1 file changed, 69 insertions(+), 16 deletions(-) diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index b52cb809cf..d23b4cc119 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -28,7 +28,13 @@ const StateToEventsMap: Record = synced: LiveObjectsEvent.synced, }; -type BatchCallback = (batchContext: BatchContext) => void; +export type LiveObjectsEventCallback = () => void; + +export interface OnLiveObjectsEventResponse { + off(): void; +} + +export type BatchCallback = (batchContext: BatchContext) => void; export class LiveObjects { private _client: BaseClient; @@ -36,7 +42,9 @@ export class LiveObjects { private _state: LiveObjectsState; // composition over inheritance since we cannot import class directly into plugin code. // instead we obtain a class type from the client - private _eventEmitter: EventEmitter; + private _eventEmitterInternal: EventEmitter; + // related to RTC10, should have a separate EventEmitter for users of the library + private _eventEmitterPublic: EventEmitter; private _liveObjectsPool: LiveObjectsPool; private _syncLiveObjectsDataPool: SyncLiveObjectsDataPool; private _currentSyncId: string | undefined; @@ -50,7 +58,8 @@ export class LiveObjects { this._channel = channel; this._client = channel.client; this._state = LiveObjectsState.initialized; - this._eventEmitter = new this._client.EventEmitter(this._client.logger); + this._eventEmitterInternal = new this._client.EventEmitter(this._client.logger); + this._eventEmitterPublic = new this._client.EventEmitter(this._client.logger); this._liveObjectsPool = new LiveObjectsPool(this); this._syncLiveObjectsDataPool = new SyncLiveObjectsDataPool(this); this._bufferedStateOperations = []; @@ -66,7 +75,7 @@ export class LiveObjects { // if we're not synced yet, wait for SYNC sequence to finish before returning root if (this._state !== LiveObjectsState.synced) { - await this._eventEmitter.once(LiveObjectsEvent.synced); + await this._eventEmitterInternal.once(LiveObjectsEvent.synced); } return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; @@ -154,6 +163,33 @@ export class LiveObjects { return counter; } + on(event: LiveObjectsEvent, callback: LiveObjectsEventCallback): OnLiveObjectsEventResponse { + // we don't require any specific channel mode to be set to call this public method + this._eventEmitterPublic.on(event, callback); + + const off = () => { + this._eventEmitterPublic.off(event, callback); + }; + + return { off }; + } + + off(event: LiveObjectsEvent, callback: LiveObjectsEventCallback): void { + // we don't require any specific channel mode to be set to call this public method + + // prevent accidentally calling .off without any arguments on an EventEmitter and removing all callbacks + if (this._client.Utils.isNil(event) && this._client.Utils.isNil(callback)) { + return; + } + + this._eventEmitterPublic.off(event, callback); + } + + offAll(): void { + // we don't require any specific channel mode to be set to call this public method + this._eventEmitterPublic.off(); + } + /** * @internal */ @@ -180,7 +216,8 @@ export class LiveObjects { */ handleStateSyncMessages(stateMessages: StateMessage[], syncChannelSerial: string | null | undefined): void { const { syncId, syncCursor } = this._parseSyncChannelSerial(syncChannelSerial); - if (this._currentSyncId !== syncId) { + const newSyncSequence = this._currentSyncId !== syncId; + if (newSyncSequence) { this._startNewSync(syncId, syncCursor); } @@ -188,7 +225,9 @@ export class LiveObjects { // if this is the last (or only) message in a sequence of sync updates, end the sync if (!syncCursor) { - this._endSync(); + // defer the state change event until the next tick if this was a new sync sequence + // to allow any event listeners to process the start of the new sequence event that was emitted earlier during this event loop. + this._endSync(newSyncSequence); } } @@ -219,14 +258,20 @@ export class LiveObjects { `channel=${this._channel.name}, hasState=${hasState}`, ); - if (hasState) { + const fromInitializedState = this._state === LiveObjectsState.initialized; + if (hasState || fromInitializedState) { + // should always start a new sync sequence if we're in the initialized state, no matter the HAS_STATE flag value. + // this guarantees we emit both "syncing" -> "synced" events in that order. this._startNewSync(); - } else { - // no HAS_STATE flag received on attach, can end SYNC sequence immediately - // and treat it as no state on a channel + } + + if (!hasState) { + // if no HAS_STATE flag received on attach, we can end SYNC sequence immediately and treat it as no state on a channel. this._liveObjectsPool.reset(); this._syncLiveObjectsDataPool.reset(); - this._endSync(); + // defer the state change event until the next tick if we started a new sequence just now due to being in initialized state. + // this allows any event listeners to process the start of the new sequence event that was emitted earlier during this event loop. + this._endSync(fromInitializedState); } } @@ -287,10 +332,10 @@ export class LiveObjects { this._syncLiveObjectsDataPool.reset(); this._currentSyncId = syncId; this._currentSyncCursor = syncCursor; - this._stateChange(LiveObjectsState.syncing); + this._stateChange(LiveObjectsState.syncing, false); } - private _endSync(): void { + private _endSync(deferStateEvent: boolean): void { this._applySync(); // should apply buffered state operations after we applied the SYNC data. // can use regular state messages application logic @@ -300,7 +345,7 @@ export class LiveObjects { this._syncLiveObjectsDataPool.reset(); this._currentSyncId = undefined; this._currentSyncCursor = undefined; - this._stateChange(LiveObjectsState.synced); + this._stateChange(LiveObjectsState.synced, deferStateEvent); } private _parseSyncChannelSerial(syncChannelSerial: string | null | undefined): { @@ -420,7 +465,7 @@ export class LiveObjects { } } - private _stateChange(state: LiveObjectsState): void { + private _stateChange(state: LiveObjectsState, deferEvent: boolean): void { if (this._state === state) { return; } @@ -431,6 +476,14 @@ export class LiveObjects { return; } - this._eventEmitter.emit(event); + if (deferEvent) { + this._client.Platform.Config.nextTick(() => { + this._eventEmitterInternal.emit(event); + this._eventEmitterPublic.emit(event); + }); + } else { + this._eventEmitterInternal.emit(event); + this._eventEmitterPublic.emit(event); + } } } From c00e9abc3cda38e77bbbfe1c2aaf198a75948d1c Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 23 Jan 2025 04:35:39 +0000 Subject: [PATCH 117/166] Refactor `LiveObject` subscription events --- src/plugins/liveobjects/liveobject.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index 084d0dd25c..bb9216dd90 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -3,8 +3,8 @@ import type EventEmitter from 'common/lib/util/eventemitter'; import { LiveObjects } from './liveobjects'; import { StateMessage, StateObject, StateOperation } from './statemessage'; -enum LiveObjectEvents { - Updated = 'Updated', +export enum LiveObjectSubscriptionEvent { + updated = 'updated', } export interface LiveObjectData { @@ -30,7 +30,7 @@ export abstract class LiveObject< TUpdate extends LiveObjectUpdate = LiveObjectUpdate, > { protected _client: BaseClient; - protected _eventEmitter: EventEmitter; + protected _subscriptions: EventEmitter; protected _objectId: string; /** * Represents an aggregated value for an object, which combines the initial value for an object from the create operation, @@ -55,7 +55,7 @@ export abstract class LiveObject< objectId: string, ) { this._client = this._liveObjects.getClient(); - this._eventEmitter = new this._client.EventEmitter(this._client.logger); + this._subscriptions = new this._client.EventEmitter(this._client.logger); this._objectId = objectId; this._dataRef = this._getZeroValueData(); // use empty timeserials vector by default, so any future operation can be applied to this object @@ -67,10 +67,10 @@ export abstract class LiveObject< subscribe(listener: (update: TUpdate) => void): SubscribeResponse { this._liveObjects.throwIfMissingStateSubscribeMode(); - this._eventEmitter.on(LiveObjectEvents.Updated, listener); + this._subscriptions.on(LiveObjectSubscriptionEvent.updated, listener); const unsubscribe = () => { - this._eventEmitter.off(LiveObjectEvents.Updated, listener); + this._subscriptions.off(LiveObjectSubscriptionEvent.updated, listener); }; return { unsubscribe }; @@ -86,12 +86,12 @@ export abstract class LiveObject< return; } - this._eventEmitter.off(LiveObjectEvents.Updated, listener); + this._subscriptions.off(LiveObjectSubscriptionEvent.updated, listener); } unsubscribeAll(): void { // can allow calling this public method without checking for state modes on the channel as the result of this method is not dependant on them - this._eventEmitter.off(LiveObjectEvents.Updated); + this._subscriptions.off(LiveObjectSubscriptionEvent.updated); } /** @@ -102,7 +102,7 @@ export abstract class LiveObject< } /** - * Emits the {@link LiveObjectEvents.Updated} event with provided update object if it isn't a noop. + * Emits the {@link LiveObjectSubscriptionEvent.updated} event with provided update object if it isn't a noop. * * @internal */ @@ -112,7 +112,7 @@ export abstract class LiveObject< return; } - this._eventEmitter.emit(LiveObjectEvents.Updated, update); + this._subscriptions.emit(LiveObjectSubscriptionEvent.updated, update); } /** From 3c40056927e87dbe3dd258905b6c24ea8b0da1ab Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 23 Jan 2025 04:41:59 +0000 Subject: [PATCH 118/166] Emit lifecycle events on `LiveObject` Currently emits `deleted` upon receiving `OBJECT_DELETE` for the object DTP-1034 --- src/plugins/liveobjects/liveobject.ts | 41 ++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index bb9216dd90..49d0796884 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -25,12 +25,23 @@ export interface SubscribeResponse { unsubscribe(): void; } +export enum LiveObjectLifecycleEvent { + deleted = 'deleted', +} + +export type LiveObjectLifecycleEventCallback = () => void; + +export interface OnLiveObjectLifecycleEventResponse { + off(): void; +} + export abstract class LiveObject< TData extends LiveObjectData = LiveObjectData, TUpdate extends LiveObjectUpdate = LiveObjectUpdate, > { protected _client: BaseClient; protected _subscriptions: EventEmitter; + protected _lifecycleEvents: EventEmitter; protected _objectId: string; /** * Represents an aggregated value for an object, which combines the initial value for an object from the create operation, @@ -56,6 +67,7 @@ export abstract class LiveObject< ) { this._client = this._liveObjects.getClient(); this._subscriptions = new this._client.EventEmitter(this._client.logger); + this._lifecycleEvents = new this._client.EventEmitter(this._client.logger); this._objectId = objectId; this._dataRef = this._getZeroValueData(); // use empty timeserials vector by default, so any future operation can be applied to this object @@ -94,6 +106,33 @@ export abstract class LiveObject< this._subscriptions.off(LiveObjectSubscriptionEvent.updated); } + on(event: LiveObjectLifecycleEvent, callback: LiveObjectLifecycleEventCallback): OnLiveObjectLifecycleEventResponse { + // we don't require any specific channel mode to be set to call this public method + this._lifecycleEvents.on(event, callback); + + const off = () => { + this._lifecycleEvents.off(event, callback); + }; + + return { off }; + } + + off(event: LiveObjectLifecycleEvent, callback: LiveObjectLifecycleEventCallback): void { + // we don't require any specific channel mode to be set to call this public method + + // prevent accidentally calling .off without any arguments on an EventEmitter and removing all callbacks + if (this._client.Utils.isNil(event) && this._client.Utils.isNil(callback)) { + return; + } + + this._lifecycleEvents.off(event, callback); + } + + offAll(): void { + // we don't require any specific channel mode to be set to call this public method + this._lifecycleEvents.off(); + } + /** * @internal */ @@ -124,7 +163,7 @@ export abstract class LiveObject< this._tombstone = true; this._tombstonedAt = Date.now(); this._dataRef = this._getZeroValueData(); - // TODO: emit "deleted" event so that end users get notified about this object getting deleted + this._lifecycleEvents.emit(LiveObjectLifecycleEvent.deleted); } /** From de71ceffc89c91b131a169a47385ff45ad2751e3 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 27 Jan 2025 03:31:52 +0000 Subject: [PATCH 119/166] Refactor maximum size of messages exceeded error message --- src/common/lib/client/realtimechannel.ts | 6 +----- src/common/lib/client/restchannel.ts | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index f7b1bd44bc..179f38e5d9 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -265,11 +265,7 @@ class RealtimeChannel extends EventEmitter { const size = getMessagesSize(messages); if (size > maxMessageSize) { throw new ErrorInfo( - 'Maximum size of messages that can be published at once exceeded ( was ' + - size + - ' bytes; limit is ' + - maxMessageSize + - ' bytes)', + `Maximum size of messages that can be published at once exceeded (was ${size} bytes; limit is ${maxMessageSize} bytes)`, 40009, 400, ); diff --git a/src/common/lib/client/restchannel.ts b/src/common/lib/client/restchannel.ts index e27133c6e7..9a5ad74efc 100644 --- a/src/common/lib/client/restchannel.ts +++ b/src/common/lib/client/restchannel.ts @@ -117,11 +117,7 @@ class RestChannel { maxMessageSize = options.maxMessageSize; if (size > maxMessageSize) { throw new ErrorInfo( - 'Maximum size of messages that can be published at once exceeded ( was ' + - size + - ' bytes; limit is ' + - maxMessageSize + - ' bytes)', + `Maximum size of messages that can be published at once exceeded (was ${size} bytes; limit is ${maxMessageSize} bytes)`, 40009, 400, ); From e6aa40eb2f97714b22e89a61b7d043c2ebd2d3a9 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 27 Jan 2025 03:33:42 +0000 Subject: [PATCH 120/166] Add missing `helper.recordPrivateApi` calls to live objects tests --- test/realtime/live_objects.test.js | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index d7bc2cce25..ca38ea5acf 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -438,6 +438,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(valuesMap.get('stringKey')).to.equal('stringValue', 'Check values map has correct string value key'); expect(valuesMap.get('emptyStringKey')).to.equal('', 'Check values map has correct empty string value key'); + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( BufferUtils.areBuffersEqual( valuesMap.get('bytesKey'), @@ -445,6 +447,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ), 'Check values map has correct bytes value key', ).to.be.true; + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( BufferUtils.areBuffersEqual(valuesMap.get('emptyBytesKey'), BufferUtils.base64Decode('')), 'Check values map has correct empty bytes value key', @@ -688,7 +692,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'can apply MAP_CREATE with primitives state operation messages', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, liveObjectsHelper, channelName, helper } = ctx; // LiveObjects public API allows us to check value of objects we've created based on MAP_CREATE ops // if we assign those objects to another map (root for example), as there is no way to access those objects from the internal pool directly. @@ -729,6 +733,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], Object.entries(fixture.entries ?? {}).forEach(([key, keyData]) => { if (keyData.data.encoding) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( BufferUtils.areBuffersEqual(mapObj.get(key), BufferUtils.base64Decode(keyData.data.value)), `Check map "${mapKey}" has correct value for "${key}" key`, @@ -902,7 +908,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'can apply MAP_SET with primitives state operation messages', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, liveObjectsHelper, channelName, helper } = ctx; // check root is empty before ops primitiveKeyData.forEach((keyData) => { @@ -929,6 +935,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check everything is applied correctly primitiveKeyData.forEach((keyData) => { if (keyData.data.encoding) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), `Check root has correct value for "${keyData.key}" key after MAP_SET op`, @@ -1785,7 +1793,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'buffered state operation messages are applied when STATE_SYNC sequence ends', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, liveObjectsHelper, channel, helper } = ctx; // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages await liveObjectsHelper.processStateObjectMessageOnChannel({ @@ -1814,6 +1822,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check everything is applied correctly primitiveKeyData.forEach((keyData) => { if (keyData.data.encoding) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), `Check root has correct value for "${keyData.key}" key after STATE_SYNC has ended and buffered operations are applied`, @@ -2013,7 +2023,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], description: 'subsequent state operation messages are applied immediately after STATE_SYNC ended and buffers are applied', action: async (ctx) => { - const { root, liveObjectsHelper, channel, channelName } = ctx; + const { root, liveObjectsHelper, channel, channelName, helper } = ctx; // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages await liveObjectsHelper.processStateObjectMessageOnChannel({ @@ -2052,6 +2062,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check buffered operations are applied, as well as the most recent operation outside of the STATE_SYNC is applied primitiveKeyData.forEach((keyData) => { if (keyData.data.encoding) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), `Check root has correct value for "${keyData.key}" key after STATE_SYNC has ended and buffered operations are applied`, @@ -2279,10 +2291,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'LiveMap.set sends MAP_SET operation with primitive values', action: async (ctx) => { - const { root } = ctx; + const { root, helper } = ctx; await Promise.all( primitiveKeyData.map(async (keyData) => { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); const value = keyData.data.encoding ? BufferUtils.base64Decode(keyData.data.value) : keyData.data.value; await root.set(keyData.key, value); }), @@ -2291,6 +2304,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // check everything is applied correctly primitiveKeyData.forEach((keyData) => { if (keyData.data.encoding) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), `Check root has correct value for "${keyData.key}" key after LiveMap.set call`, @@ -2602,12 +2617,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'LiveObjects.createMap sends MAP_CREATE operation with primitive values', action: async (ctx) => { - const { liveObjects } = ctx; + const { liveObjects, helper } = ctx; const maps = await Promise.all( primitiveMapsFixtures.map(async (mapFixture) => { const entries = mapFixture.entries ? Object.entries(mapFixture.entries).reduce((acc, [key, keyData]) => { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); const value = keyData.data.encoding ? BufferUtils.base64Decode(keyData.data.value) : keyData.data.value; @@ -2634,6 +2650,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], Object.entries(fixture.entries ?? {}).forEach(([key, keyData]) => { if (keyData.data.encoding) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( BufferUtils.areBuffersEqual(map.get(key), BufferUtils.base64Decode(keyData.data.value)), `Check map #${i + 1} has correct value for "${key}" key`, From 85d988c4e89c257b171a42cb64032037c7271f3e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 27 Jan 2025 03:32:17 +0000 Subject: [PATCH 121/166] Update `Utils.dataSizeBytes` function to include numbers, booleans and more generic Buffer type This is in preparation for message size calculation for StateMessages, which can have numbers and booleans as user provided data [1]. Also should use platform agnostic `Bufferlike` type provided by the Platform module to have better support for node.js/browser buffers. [1] https://ably.atlassian.net/browse/DTP-1118 --- src/common/lib/util/utils.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/common/lib/util/utils.ts b/src/common/lib/util/utils.ts index b88f9dd4b3..6daae712cb 100644 --- a/src/common/lib/util/utils.ts +++ b/src/common/lib/util/utils.ts @@ -1,4 +1,4 @@ -import Platform from 'common/platform'; +import Platform, { Bufferlike } from 'common/platform'; import ErrorInfo, { PartialErrorInfo } from 'common/lib/types/errorinfo'; import { ModularPlugins } from '../client/modularplugins'; import { MsgPack } from 'common/types/msgpack'; @@ -279,15 +279,23 @@ export function inspectBody(body: unknown): string { } } -/* Data is assumed to be either a string or a buffer. */ -export function dataSizeBytes(data: string | Buffer): number { +/* Data is assumed to be either a string, a number, a boolean or a buffer. */ +export function dataSizeBytes(data: string | number | boolean | Bufferlike): number { if (Platform.BufferUtils.isBuffer(data)) { return Platform.BufferUtils.byteLength(data); } if (typeof data === 'string') { return Platform.Config.stringByteSize(data); } - throw new Error('Expected input of Utils.dataSizeBytes to be a buffer or string, but was: ' + typeof data); + if (typeof data === 'number') { + return 8; + } + if (typeof data === 'boolean') { + return 1; + } + throw new Error( + `Expected input of Utils.dataSizeBytes to be a string, a number, a boolean or a buffer, but was: ${typeof data}`, + ); } export function cheapRandStr(): string { From 2b600246896319a8ca2ccf5c55aecd47affaeeab Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 27 Jan 2025 03:30:54 +0000 Subject: [PATCH 122/166] Apply `ConnectionDetails.maxMessageSize` limit when publishing state messages Resolves DTP-1118 --- src/common/lib/client/defaultrealtime.ts | 2 + src/plugins/liveobjects/liveobjects.ts | 9 + src/plugins/liveobjects/statemessage.ts | 117 ++++++++ test/common/modules/private_api_recorder.js | 6 +- test/realtime/live_objects.test.js | 285 +++++++++++++++++++- 5 files changed, 415 insertions(+), 4 deletions(-) diff --git a/src/common/lib/client/defaultrealtime.ts b/src/common/lib/client/defaultrealtime.ts index 77854711a1..7b19c570e4 100644 --- a/src/common/lib/client/defaultrealtime.ts +++ b/src/common/lib/client/defaultrealtime.ts @@ -19,6 +19,7 @@ import { import { Http } from 'common/types/http'; import Defaults from '../util/defaults'; import Logger from '../util/logger'; +import { MessageEncoding } from '../types/message'; /** `DefaultRealtime` is the class that the non tree-shakable version of the SDK exports as `Realtime`. It ensures that this version of the SDK includes all of the functionality which is optionally available in the tree-shakable version. @@ -71,4 +72,5 @@ export class DefaultRealtime extends BaseRealtime { // Used by tests static _Http = Http; static _PresenceMap = PresenceMap; + static _MessageEncoding = MessageEncoding; } diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 3d6286147c..46108a77d8 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -250,6 +250,15 @@ export class LiveObjects { } stateMessages.forEach((x) => StateMessage.encode(x, this._client.MessageEncoding)); + const maxMessageSize = this._client.options.maxMessageSize; + const size = stateMessages.reduce((acc, msg) => acc + msg.getMessageSize(), 0); + if (size > maxMessageSize) { + throw new this._client.ErrorInfo( + `Maximum size of state messages that can be published at once exceeded (was ${size} bytes; limit is ${maxMessageSize} bytes)`, + 40009, + 400, + ); + } return this._channel.sendState(stateMessages); } diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/liveobjects/statemessage.ts index 77fdc5cba8..8bceed1e24 100644 --- a/src/plugins/liveobjects/statemessage.ts +++ b/src/plugins/liveobjects/statemessage.ts @@ -445,4 +445,121 @@ export class StateMessage { return result; } + + getMessageSize(): number { + let size = 0; + + size += this.clientId?.length ?? 0; + if (this.operation) { + size += this._getStateOperationSize(this.operation); + } + if (this.object) { + size += this._getStateObjectSize(this.object); + } + if (this.extras) { + size += JSON.stringify(this.extras).length; + } + + return size; + } + + private _getStateOperationSize(operation: StateOperation): number { + let size = 0; + + if (operation.mapOp) { + size += this._getStateMapOpSize(operation.mapOp); + } + if (operation.counterOp) { + size += this._getStateCounterOpSize(operation.counterOp); + } + if (operation.map) { + size += this._getStateMapSize(operation.map); + } + if (operation.counter) { + size += this._getStateCounterSize(operation.counter); + } + + return size; + } + + private _getStateObjectSize(obj: StateObject): number { + let size = 0; + + if (obj.map) { + size += this._getStateMapSize(obj.map); + } + if (obj.counter) { + size += this._getStateCounterSize(obj.counter); + } + if (obj.createOp) { + size += this._getStateOperationSize(obj.createOp); + } + + return size; + } + + private _getStateMapSize(map: StateMap): number { + let size = 0; + + Object.entries(map.entries ?? {}).forEach(([key, entry]) => { + size += key?.length ?? 0; + if (entry) { + size += this._getStateMapEntrySize(entry); + } + }); + + return size; + } + + private _getStateCounterSize(counter: StateCounter): number { + if (counter.count == null) { + return 0; + } + + return 8; + } + + private _getStateMapEntrySize(entry: StateMapEntry): number { + let size = 0; + + if (entry.data) { + size += this._getStateDataSize(entry.data); + } + + return size; + } + + private _getStateMapOpSize(mapOp: StateMapOp): number { + let size = 0; + + size += mapOp.key?.length ?? 0; + + if (mapOp.data) { + size += this._getStateDataSize(mapOp.data); + } + + return size; + } + + private _getStateCounterOpSize(operation: StateCounterOp): number { + if (operation.amount == null) { + return 0; + } + + return 8; + } + + private _getStateDataSize(data: StateData): number { + let size = 0; + + if (data.value) { + size += this._getStateValueSize(data.value); + } + + return size; + } + + private _getStateValueSize(value: StateValue): number { + return this._utils.dataSizeBytes(value); + } } diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index 2898aed40b..f1f07a2e17 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -16,6 +16,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'call.Defaults.getPort', 'call.Defaults.normaliseOptions', 'call.EventEmitter.emit', + 'call.LiveObject.getObjectId', 'call.LiveObject.isTombstoned', 'call.LiveObjects._liveObjectsPool._onGCInterval', 'call.LiveObjects._liveObjectsPool.get', @@ -25,7 +26,11 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'call.Platform.nextTick', 'call.PresenceMessage.fromValues', 'call.ProtocolMessage.setFlag', + 'call.StateMessage.encode', + 'call.StateMessage.fromValues', + 'call.StateMessage.getMessageSize', 'call.Utils.copy', + 'call.Utils.dataSizeBytes', 'call.Utils.getRetryTime', 'call.Utils.inspectError', 'call.Utils.keysArray', @@ -47,7 +52,6 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'call.http._getHosts', 'call.http.checkConnectivity', 'call.http.doUri', - 'call.LiveObject.getObjectId', 'call.msgpack.decode', 'call.msgpack.encode', 'call.presence._myMembers.put', diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index ca38ea5acf..f98089bf7e 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -9,6 +9,8 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ) { const expect = chai.expect; const BufferUtils = Ably.Realtime.Platform.BufferUtils; + const Utils = Ably.Realtime.Utils; + const MessageEncoding = Ably.Realtime._MessageEncoding; const createPM = Ably.makeProtocolMessageFromDeserialized({ LiveObjectsPlugin }); const liveObjectsFixturesChannel = 'liveobjects_fixtures'; const nextTick = Ably.Realtime.Platform.Config.nextTick; @@ -58,14 +60,20 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], } async function expectToThrowAsync(fn, errorStr) { - let verifiedError = false; + let savedError; try { await fn(); } catch (error) { expect(error.message).to.have.string(errorStr); - verifiedError = true; + savedError = error; } - expect(verifiedError, 'Expected async function to throw an error').to.be.true; + expect(savedError, 'Expected async function to throw an error').to.exist; + + return savedError; + } + + function stateMessageFromValues(values) { + return LiveObjectsPlugin.StateMessage.fromValues(values, Utils, MessageEncoding); } describe('realtime/live_objects', function () { @@ -3983,6 +3991,277 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await scenario.action({ liveObjects, liveObjectsHelper, channelName, channel, root, map, counter, helper }); }, client); }); + + /** + * @spec TO3l8 + * @spec RSL1i + */ + it('state message publish respects connectionDetails.maxMessageSize', async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper, { clientId: 'test' }); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + await client.connection.once('connected'); + + const connectionManager = client.connection.connectionManager; + const connectionDetails = connectionManager.connectionDetails; + const connectionDetailsPromise = connectionManager.once('connectiondetails'); + + helper.recordPrivateApi('write.connectionManager.connectionDetails.maxMessageSize'); + connectionDetails.maxMessageSize = 64; + + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + helper.recordPrivateApi('call.transport.onProtocolMessage'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + // forge lower maxMessageSize + connectionManager.activeProtocol.getTransport().onProtocolMessage( + createPM({ + action: 4, // CONNECTED + connectionDetails, + }), + ); + + helper.recordPrivateApi('listen.connectionManager.connectiondetails'); + await connectionDetailsPromise; + + const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + const data = new Array(100).fill('a').join(''); + const error = await expectRejectedWith( + async () => root.set('key', data), + 'Maximum size of state messages that can be published at once exceeded', + ); + + expect(error.code).to.equal(40009, 'Check maximum size of messages error has correct error code'); + }, client); + }); + + describe('StateMessage message size', () => { + const stateMessageSizeScenarios = [ + { + description: 'client id', + message: stateMessageFromValues({ + clientId: 'my-client', + }), + expected: Utils.dataSizeBytes('my-client'), + }, + { + description: 'extras', + message: stateMessageFromValues({ + extras: { foo: 'bar' }, + }), + expected: Utils.dataSizeBytes('{"foo":"bar"}'), + }, + { + description: 'object id', + message: stateMessageFromValues({ + operation: { objectId: 'object-id' }, + }), + expected: 0, + }, + { + description: 'map create op no payload', + message: stateMessageFromValues({ + operation: { action: 0, objectId: 'object-id' }, + }), + expected: 0, + }, + { + description: 'map create op with object payload', + message: stateMessageFromValues( + { + operation: { + action: 0, + objectId: 'object-id', + map: { + semantics: 0, + entries: { 'key-1': { tombstone: false, data: { objectId: 'another-object-id' } } }, + }, + }, + }, + MessageEncoding, + ), + expected: Utils.dataSizeBytes('key-1'), + }, + { + description: 'map create op with string payload', + message: stateMessageFromValues( + { + operation: { + action: 0, + objectId: 'object-id', + map: { semantics: 0, entries: { 'key-1': { tombstone: false, data: { value: 'a string' } } } }, + }, + }, + MessageEncoding, + ), + expected: Utils.dataSizeBytes('key-1') + Utils.dataSizeBytes('a string'), + }, + { + description: 'map create op with bytes payload', + message: stateMessageFromValues( + { + operation: { + action: 0, + objectId: 'object-id', + map: { + semantics: 0, + entries: { 'key-1': { tombstone: false, data: { value: BufferUtils.utf8Encode('my-value') } } }, + }, + }, + }, + MessageEncoding, + ), + expected: Utils.dataSizeBytes('key-1') + Utils.dataSizeBytes(BufferUtils.utf8Encode('my-value')), + }, + { + description: 'map create op with boolean payload', + message: stateMessageFromValues( + { + operation: { + action: 0, + objectId: 'object-id', + map: { semantics: 0, entries: { 'key-1': { tombstone: false, data: { value: true } } } }, + }, + }, + MessageEncoding, + ), + expected: Utils.dataSizeBytes('key-1') + 1, + }, + { + description: 'map remove op', + message: stateMessageFromValues({ + operation: { action: 2, objectId: 'object-id', mapOp: { key: 'my-key' } }, + }), + expected: Utils.dataSizeBytes('my-key'), + }, + { + description: 'map set operation value=object', + message: stateMessageFromValues({ + operation: { + action: 1, + objectId: 'object-id', + mapOp: { key: 'my-key', data: { objectId: 'another-object-id' } }, + }, + }), + expected: Utils.dataSizeBytes('my-key'), + }, + { + description: 'map set operation value=string', + message: stateMessageFromValues({ + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { value: 'my-value' } } }, + }), + expected: Utils.dataSizeBytes('my-key') + Utils.dataSizeBytes('my-value'), + }, + { + description: 'map set operation value=bytes', + message: stateMessageFromValues({ + operation: { + action: 1, + objectId: 'object-id', + mapOp: { key: 'my-key', data: { value: BufferUtils.utf8Encode('my-value') } }, + }, + }), + expected: Utils.dataSizeBytes('my-key') + Utils.dataSizeBytes(BufferUtils.utf8Encode('my-value')), + }, + { + description: 'map set operation value=boolean', + message: stateMessageFromValues({ + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { value: true } } }, + }), + expected: Utils.dataSizeBytes('my-key') + 1, + }, + { + description: 'map set operation value=double', + message: stateMessageFromValues({ + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { value: 123.456 } } }, + }), + expected: Utils.dataSizeBytes('my-key') + 8, + }, + { + description: 'map object', + message: stateMessageFromValues({ + object: { + objectId: 'object-id', + map: { + semantics: 0, + entries: { + 'key-1': { tombstone: false, data: { value: 'a string' } }, + 'key-2': { tombstone: true, data: { value: 'another string' } }, + }, + }, + createOp: { + action: 0, + objectId: 'object-id', + map: { semantics: 0, entries: { 'key-3': { tombstone: false, data: { value: 'third string' } } } }, + }, + siteTimeserials: { aaa: lexicoTimeserial('aaa', 111, 111, 1) }, // shouldn't be counted + tombstone: false, + }, + }), + expected: + Utils.dataSizeBytes('key-1') + + Utils.dataSizeBytes('a string') + + Utils.dataSizeBytes('key-2') + + Utils.dataSizeBytes('another string') + + Utils.dataSizeBytes('key-3') + + Utils.dataSizeBytes('third string'), + }, + { + description: 'counter create op no payload', + message: stateMessageFromValues({ + operation: { action: 3, objectId: 'object-id' }, + }), + expected: 0, + }, + { + description: 'counter create op with payload', + message: stateMessageFromValues({ + operation: { action: 3, objectId: 'object-id', counter: { count: 1234567 } }, + }), + expected: 8, + }, + { + description: 'counter inc op', + message: stateMessageFromValues({ + operation: { action: 4, objectId: 'object-id', counterOp: { amount: 123.456 } }, + }), + expected: 8, + }, + { + description: 'counter object', + message: stateMessageFromValues({ + object: { + objectId: 'object-id', + counter: { count: 1234567 }, + createOp: { + action: 3, + objectId: 'object-id', + counter: { count: 9876543 }, + }, + siteTimeserials: { aaa: lexicoTimeserial('aaa', 111, 111, 1) }, // shouldn't be counted + tombstone: false, + }, + }), + expected: 8 + 8, + }, + ]; + + /** @nospec */ + forScenarios(stateMessageSizeScenarios, function (helper, scenario) { + helper.recordPrivateApi('call.StateMessage.encode'); + LiveObjectsPlugin.StateMessage.encode(scenario.message); + helper.recordPrivateApi('call.BufferUtils.utf8Encode'); // was called by a scenario to create buffers + helper.recordPrivateApi('call.StateMessage.fromValues'); // was called by a scenario to create a StateMessage instance + helper.recordPrivateApi('call.Utils.dataSizeBytes'); // was called by a scenario to calculated the expected byte size + helper.recordPrivateApi('call.StateMessage.getMessageSize'); + expect(scenario.message.getMessageSize()).to.equal(scenario.expected); + }); + }); }); /** @nospec */ From d50ea2d38e0be3c75bb9ffecde250e99c8d6b785 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 12 Feb 2025 08:43:09 +0000 Subject: [PATCH 123/166] Fix test failures channelSerial test should have always expected `channelSerial` value, but was wrongly set to `channelSerial2` in https://github.com/ably/ably-js/pull/1961. live_objects connectionDetails.maxMessageSize test wrongly used obsolete `expectRejectedWith` function due to PR merge order. --- test/realtime/channel.test.js | 2 +- test/realtime/live_objects.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/realtime/channel.test.js b/test/realtime/channel.test.js index 1bdf114ee5..0323f403a1 100644 --- a/test/realtime/channel.test.js +++ b/test/realtime/channel.test.js @@ -1882,7 +1882,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async transport.send = function (msg) { if (msg.action === 10) { try { - expect(msg.channelSerial).to.equal('channelSerial2'); + expect(msg.channelSerial).to.equal('channelSerial'); resolve(); } catch (error) { reject(error); diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index c0d71d23eb..9cfa0ea887 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -4246,7 +4246,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const root = await liveObjects.getRoot(); const data = new Array(100).fill('a').join(''); - const error = await expectRejectedWith( + const error = await expectToThrowAsync( async () => root.set('key', data), 'Maximum size of state messages that can be published at once exceeded', ); From b23d31c645d8a77a05dc740588abd81be1490750 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 31 Jan 2025 04:37:42 +0000 Subject: [PATCH 124/166] Refactor `SharedHelper.restTestOnJsonMsgpack` Change function name to `testOnJsonMsgpack`. Refactor the function to be more generic and appropriate to use with Realtime clients. This change was originally in preparation for DTP-1096, but it was decided to further extend LiveObjects tests to also be run on multiple transports and protocols using `SharedHelper.testOnAllTransports` function. This refactoring change is still preserved as it provides an overall improvement to the test suite. --- test/common/modules/shared_helper.js | 27 ++++++++++++++++----------- test/rest/history.test.js | 15 ++++++++++----- test/rest/request.test.js | 18 ++++++++++++------ test/rest/status.test.js | 3 ++- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/test/common/modules/shared_helper.js b/test/common/modules/shared_helper.js index 8cafaef289..4b46f451d0 100644 --- a/test/common/modules/shared_helper.js +++ b/test/common/modules/shared_helper.js @@ -373,17 +373,18 @@ define([ itFn(name + '_with_text_transport', createTest({ transports, useBinaryProtocol: false })); } - static restTestOnJsonMsgpack(name, testFn, skip) { - var itFn = skip ? it.skip : it; - itFn(name + ' with binary protocol', async function () { - const helper = this.test.helper.withParameterisedTestTitle(name); + static testOnJsonMsgpack(testName, testFn, skip, only) { + const itFn = skip ? it.skip : only ? it.only : it; - await testFn(new clientModule.AblyRest(helper, { useBinaryProtocol: true }), name + '_binary', helper); + itFn(testName + ' with binary protocol', async function () { + const helper = this.test.helper.withParameterisedTestTitle(testName); + const channelName = testName + ' binary'; + await testFn({ useBinaryProtocol: true }, channelName, helper); }); - itFn(name + ' with text protocol', async function () { - const helper = this.test.helper.withParameterisedTestTitle(name); - - await testFn(new clientModule.AblyRest(helper, { useBinaryProtocol: false }), name + '_text', helper); + itFn(testName + ' with text protocol', async function () { + const helper = this.test.helper.withParameterisedTestTitle(testName); + const channelName = testName + ' text'; + await testFn({ useBinaryProtocol: false }, channelName, helper); }); } @@ -497,8 +498,12 @@ define([ SharedHelper.testOnAllTransports(thisInDescribe, name, testFn, true); }; - SharedHelper.restTestOnJsonMsgpack.skip = function (name, testFn) { - SharedHelper.restTestOnJsonMsgpack(name, testFn, true); + SharedHelper.testOnJsonMsgpack.skip = function (testName, testFn) { + SharedHelper.testOnJsonMsgpack(testName, testFn, true); + }; + + SharedHelper.testOnJsonMsgpack.only = function (testName, testFn) { + SharedHelper.testOnJsonMsgpack(testName, testFn, false, true); }; return (module.exports = SharedHelper); diff --git a/test/rest/history.test.js b/test/rest/history.test.js index bc26ebbc0f..ce58245969 100644 --- a/test/rest/history.test.js +++ b/test/rest/history.test.js @@ -31,7 +31,8 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * @spec RSL2 * @spec RSL2a */ - Helper.restTestOnJsonMsgpack('history_simple', async function (rest, channelName, helper) { + Helper.testOnJsonMsgpack('history_simple', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); var testchannel = rest.channels.get('persisted:' + channelName); /* first, send a number of events to this channel */ @@ -63,7 +64,8 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * @spec RSL2 * @spec RSL2a */ - Helper.restTestOnJsonMsgpack('history_multiple', async function (rest, channelName, helper) { + Helper.testOnJsonMsgpack('history_multiple', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); var testchannel = rest.channels.get('persisted:' + channelName); /* first, send a number of events to this channel */ @@ -92,7 +94,8 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * @spec RSL2b2 * @specpartial RSL2b3 - should also test maximum supported limit of 1000 */ - Helper.restTestOnJsonMsgpack('history_simple_paginated_b', async function (rest, channelName, helper) { + Helper.testOnJsonMsgpack('history_simple_paginated_b', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); var testchannel = rest.channels.get('persisted:' + channelName); /* first, send a number of events to this channel */ @@ -259,7 +262,8 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { }); /** @nospec */ - Helper.restTestOnJsonMsgpack('history_encoding_errors', async function (rest, channelName) { + Helper.testOnJsonMsgpack('history_encoding_errors', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); var testchannel = rest.channels.get('persisted:' + channelName); var badMessage = { name: 'jsonUtf8string', encoding: 'json/utf-8', data: '{"foo":"bar"}' }; testchannel.publish(badMessage); @@ -272,7 +276,8 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { }); /** @specpartial TG4 - in the context of RestChannel#history */ - Helper.restTestOnJsonMsgpack('history_no_next_page', async function (rest, channelName) { + Helper.testOnJsonMsgpack('history_no_next_page', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); const channel = rest.channels.get(channelName); const firstPage = await channel.history(); diff --git a/test/rest/request.test.js b/test/rest/request.test.js index 7132b0c5ef..c54a7f7430 100644 --- a/test/rest/request.test.js +++ b/test/rest/request.test.js @@ -28,7 +28,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @specpartial RSC7e - tests providing a version value in .request parameters * @specpartial CSV2c - tests version is provided in http requests */ - Helper.restTestOnJsonMsgpack('request_version', function (rest, _, helper) { + Helper.testOnJsonMsgpack('request_version', function (options, _, helper) { + const rest = helper.AblyRest(options); const version = 150; // arbitrarily chosen async function testRequestHandler(_, __, headers) { @@ -56,7 +57,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec HP5 * @specpartial RSC19f - basic test for passing a http method, path and version parameters */ - Helper.restTestOnJsonMsgpack('request_time', async function (rest) { + Helper.testOnJsonMsgpack('request_time', async function (options, _, helper) { + const rest = helper.AblyRest(options); const res = await rest.request('get', '/time', 3, null, null, null); expect(res.statusCode).to.equal(200, 'Check statusCode'); expect(res.success).to.equal(true, 'Check success'); @@ -72,7 +74,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec HP6 * @spec HP7 */ - Helper.restTestOnJsonMsgpack('request_404', async function (rest) { + Helper.testOnJsonMsgpack('request_404', async function (options, _, helper) { + const rest = helper.AblyRest(options); /* NB: can't just use /invalid or something as the CORS preflight will * fail. Need something superficially a valid path but where the actual * request fails */ @@ -110,7 +113,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @specpartial HP2 - tests overriden .next method only * @specpartial RSC19f - more tests with passing other methods, body and parameters */ - Helper.restTestOnJsonMsgpack('request_post_get_messages', async function (rest, channelName) { + Helper.testOnJsonMsgpack('request_post_get_messages', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); var channelPath = '/channels/' + channelName + '/messages', msgone = { name: 'faye', data: 'whittaker' }, msgtwo = { name: 'martin', data: 'reed' }; @@ -155,7 +159,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec HP7 * @specpartial RSC19f - more tests with POST method and passing body */ - Helper.restTestOnJsonMsgpack('request_batch_api_success', async function (rest, name) { + Helper.testOnJsonMsgpack('request_batch_api_success', async function (options, name, helper) { + const rest = helper.AblyRest(options); var body = { channels: [name + '1', name + '2'], messages: { data: 'foo' } }; const res = await rest.request('POST', '/messages', 2, {}, body, {}); @@ -186,7 +191,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @specpartial RSC19f - more tests with POST method and passing body * @specskip */ - Helper.restTestOnJsonMsgpack.skip('request_batch_api_partial_success', async function (rest, name) { + Helper.testOnJsonMsgpack.skip('request_batch_api_partial_success', async function (options, name, helper) { + const rest = helper.AblyRest(options); var body = { channels: [name, '[invalid', ''], messages: { data: 'foo' } }; var res = await rest.request('POST', '/messages', 2, {}, body, {}); diff --git a/test/rest/status.test.js b/test/rest/status.test.js index c1e52a20b4..d138af6e6c 100644 --- a/test/rest/status.test.js +++ b/test/rest/status.test.js @@ -34,7 +34,8 @@ define(['shared_helper', 'chai'], function (Helper, chai) { * @spec CHM2e * @spec CHM2f */ - Helper.restTestOnJsonMsgpack('status0', async function (rest) { + Helper.testOnJsonMsgpack('status0', async function (options, _, helper) { + const rest = helper.AblyRest(options); var channel = rest.channels.get('status0'); var channelDetails = await channel.status(); expect(channelDetails.channelId).to.equal('status0'); From 48cb5efb751c4e7324cb7bfb9ba0e36f56c6ea76 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 12 Feb 2025 05:12:19 +0000 Subject: [PATCH 125/166] Refactor `SharedHelper.testOnAllTransports` function Change function name to `testOnAllTransportsAndProtocols` to reflect that it also runs tests on all available protocols (msgpack and JSON). Refactor to support `only` parameter, provide a test specific channel name for the test function, and support both callback and async based test functions. This makes this helper function more generic and more convenient to be used in LiveObjects tests. This change is in preparation for DTP-1096 --- test/common/modules/shared_helper.js | 50 ++++++---- test/realtime/auth.test.js | 14 +-- test/realtime/channel.test.js | 138 ++++++++++++++------------- test/realtime/crypto.test.js | 6 +- test/realtime/failure.test.js | 2 +- test/realtime/message.test.js | 6 +- test/realtime/reauth.test.js | 2 +- test/realtime/resume.test.js | 10 +- 8 files changed, 125 insertions(+), 103 deletions(-) diff --git a/test/common/modules/shared_helper.js b/test/common/modules/shared_helper.js index 4b46f451d0..46911315a2 100644 --- a/test/common/modules/shared_helper.js +++ b/test/common/modules/shared_helper.js @@ -342,35 +342,49 @@ define([ ); } - /* testFn is assumed to be a function of realtimeOptions that returns a mocha test */ - static testOnAllTransports(thisInDescribe, name, testFn, skip) { - const helper = this.forTestDefinition(thisInDescribe, name).addingHelperFunction('testOnAllTransports'); - var itFn = skip ? it.skip : it; + /* testFn is assumed to be a function of realtimeOptions and channelName that returns a mocha test */ + static testOnAllTransportsAndProtocols(thisInDescribe, testName, testFn, skip, only) { + const helper = SharedHelper.forTestDefinition(thisInDescribe, testName).addingHelperFunction( + 'testOnAllTransportsAndProtocols', + ); + const itFn = skip ? it.skip : only ? it.only : it; - function createTest(options) { + function createTest(options, channelName) { return function (done) { - this.test.helper = this.test.helper.withParameterisedTestTitle(name); - return testFn(options).apply(this, [done]); + this.test.helper = this.test.helper.withParameterisedTestTitle(testName); + // we want to support both callback-based and async test functions here. + // for this we check the return type of the test function to see if it is a Promise. + // if it is, then the test function provided is an async one, and won't call done function on its own. + // instead we attach own .then and .catch callbacks for the promise here and call done when needed + const testFnReturn = testFn(options, channelName).apply(this, [done]); + if (testFnReturn instanceof Promise) { + testFnReturn.then(done).catch(done); + } else { + return testFnReturn; + } }; } - let transports = helper.availableTransports; + const transports = helper.availableTransports; transports.forEach(function (transport) { itFn( - name + '_with_' + transport + '_binary_transport', - createTest({ transports: [transport], useBinaryProtocol: true }), + testName + ' with ' + transport + ' binary protocol', + createTest({ transports: [transport], useBinaryProtocol: true }, `${testName} ${transport} binary`), ); itFn( - name + '_with_' + transport + '_text_transport', - createTest({ transports: [transport], useBinaryProtocol: false }), + testName + ' with ' + transport + ' text protocol', + createTest({ transports: [transport], useBinaryProtocol: false }, `${testName} ${transport} text`), ); }); /* Plus one for no transport specified (ie use websocket/base mechanism if * present). (we explicitly specify all transports since node only does * websocket+nodecomet if comet is explicitly requested) * */ - itFn(name + '_with_binary_transport', createTest({ transports, useBinaryProtocol: true })); - itFn(name + '_with_text_transport', createTest({ transports, useBinaryProtocol: false })); + itFn( + testName + ' with binary protocol', + createTest({ transports, useBinaryProtocol: true }, `${testName} binary`), + ); + itFn(testName + ' with text protocol', createTest({ transports, useBinaryProtocol: false }, `${testName} text`)); } static testOnJsonMsgpack(testName, testFn, skip, only) { @@ -494,8 +508,12 @@ define([ } } - SharedHelper.testOnAllTransports.skip = function (thisInDescribe, name, testFn) { - SharedHelper.testOnAllTransports(thisInDescribe, name, testFn, true); + SharedHelper.testOnAllTransportsAndProtocols.skip = function (thisInDescribe, testName, testFn) { + SharedHelper.testOnAllTransportsAndProtocols(thisInDescribe, testName, testFn, true); + }; + + SharedHelper.testOnAllTransportsAndProtocols.only = function (thisInDescribe, testName, testFn) { + SharedHelper.testOnAllTransportsAndProtocols(thisInDescribe, testName, testFn, false, true); }; SharedHelper.testOnJsonMsgpack.skip = function (testName, testFn) { diff --git a/test/realtime/auth.test.js b/test/realtime/auth.test.js index 3cdd43dcbc..6af1a7a319 100644 --- a/test/realtime/auth.test.js +++ b/test/realtime/auth.test.js @@ -774,7 +774,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RSA4b1 * @specpartial RSA4b - token expired */ - Helper.testOnAllTransports(this, 'auth_token_expires', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'auth_token_expires', function (realtimeOpts) { return function (done) { var helper = this.test.helper, clientRealtime, @@ -879,7 +879,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @specpartial RTN15a - attempt to reconnect and restore the connection state on token expire * @specpartial RSA10e - obtain new token from authcallback when previous expires */ - Helper.testOnAllTransports(this, 'auth_tokenDetails_expiry_with_authcallback', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'auth_tokenDetails_expiry_with_authcallback', function (realtimeOpts) { return function (done) { var helper = this.test.helper, realtime, @@ -928,7 +928,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @specpartial RTN15a - attempt to reconnect and restore the connection state on token expire * @specpartial RSA10e - obtain new token from authcallback when previous expires */ - Helper.testOnAllTransports(this, 'auth_token_string_expiry_with_authcallback', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'auth_token_string_expiry_with_authcallback', function (realtimeOpts) { return function (done) { var helper = this.test.helper, realtime, @@ -975,7 +975,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RSA4a * @spec RSA4a2 */ - Helper.testOnAllTransports(this, 'auth_token_string_expiry_with_token', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'auth_token_string_expiry_with_token', function (realtimeOpts) { return function (done) { var helper = this.test.helper, realtime, @@ -1023,7 +1023,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RSA4a * @spec RSA4a2 */ - Helper.testOnAllTransports(this, 'auth_expired_token_string', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'auth_expired_token_string', function (realtimeOpts) { return function (done) { var helper = this.test.helper, realtime, @@ -1073,7 +1073,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTC8 * @specskip */ - Helper.testOnAllTransports.skip(this, 'reauth_authCallback', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols.skip(this, 'reauth_authCallback', function (realtimeOpts) { return function (done) { var helper = this.test.helper, realtime, @@ -1586,7 +1586,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); /** @nospec */ - Helper.testOnAllTransports(this, 'authorize_immediately_after_init', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'authorize_immediately_after_init', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var realtime = helper.AblyRealtime({ diff --git a/test/realtime/channel.test.js b/test/realtime/channel.test.js index 0323f403a1..a342b5a5dc 100644 --- a/test/realtime/channel.test.js +++ b/test/realtime/channel.test.js @@ -170,7 +170,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTS3c * @spec RTL16 */ - Helper.testOnAllTransports(this, 'channelinit0', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelinit0', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -208,7 +208,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @spec RTL4 */ - Helper.testOnAllTransports(this, 'channelattach0', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattach0', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -235,7 +235,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @spec RTL4 */ - Helper.testOnAllTransports(this, 'channelattach2', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattach2', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -262,7 +262,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL4 * @spec RTL5 */ - Helper.testOnAllTransports( + Helper.testOnAllTransportsAndProtocols( this, 'channelattach3', function (realtimeOpts) { @@ -303,7 +303,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @spec RTL4d */ - Helper.testOnAllTransports(this, 'channelattachempty', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattachempty', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -338,7 +338,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @spec RTL4d */ - Helper.testOnAllTransports(this, 'channelattachinvalid', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattachinvalid', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -379,7 +379,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @spec RTL6 */ - Helper.testOnAllTransports(this, 'publish_no_attach', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'publish_no_attach', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -409,7 +409,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @specpartial RTL6b - callback which is called with an error */ - Helper.testOnAllTransports(this, 'channelattach_publish_invalid', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattach_publish_invalid', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -443,7 +443,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @nospec */ - Helper.testOnAllTransports(this, 'channelattach_invalid_twice', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattach_invalid_twice', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -538,7 +538,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL4k1 * @spec RTL4m */ - Helper.testOnAllTransports(this, 'attachWithChannelParamsBasicChannelsGet', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'attachWithChannelParamsBasicChannelsGet', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'attachWithChannelParamsBasicChannelsGet'; @@ -601,7 +601,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL4m * @spec RTL16 */ - Helper.testOnAllTransports(this, 'attachWithChannelParamsBasicSetOptions', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'attachWithChannelParamsBasicSetOptions', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'attachWithChannelParamsBasicSetOptions'; @@ -656,7 +656,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL16 * @spec RTL7c */ - Helper.testOnAllTransports(this, 'subscribeAfterSetOptions', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'subscribeAfterSetOptions', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'subscribeAfterSetOptions'; @@ -732,7 +732,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); /** @spec RTL16a */ - Helper.testOnAllTransports(this, 'setOptionsCallbackBehaviour', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'setOptionsCallbackBehaviour', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'setOptionsCallbackBehaviour'; @@ -814,61 +814,65 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * Verify modes is ignored when params.modes is present * @nospec */ - Helper.testOnAllTransports(this, 'attachWithChannelParamsModesAndChannelModes', function (realtimeOpts) { - return function (done) { - const helper = this.test.helper; - var testName = 'attachWithChannelParamsModesAndChannelModes'; - try { - var realtime = helper.AblyRealtime(realtimeOpts); - realtime.connection.on('connected', function () { - var paramsModes = ['presence', 'subscribe']; - var params = { - modes: paramsModes.join(','), - }; - var channelOptions = { - params: params, - modes: ['publish', 'presence_subscribe'], - }; - var channel = realtime.channels.get(testName, channelOptions); - Helper.whenPromiseSettles(channel.attach(), function (err) { - if (err) { - helper.closeAndFinish(done, realtime, err); - return; - } - try { - helper.recordPrivateApi('read.channel.channelOptions'); - expect(channel.channelOptions).to.deep.equal(channelOptions, 'Check requested channel options'); - expect(channel.params).to.deep.equal(params, 'Check result params'); - expect(channel.modes).to.deep.equal(paramsModes, 'Check result modes'); - } catch (err) { - helper.closeAndFinish(done, realtime, err); - return; - } + Helper.testOnAllTransportsAndProtocols( + this, + 'attachWithChannelParamsModesAndChannelModes', + function (realtimeOpts) { + return function (done) { + const helper = this.test.helper; + var testName = 'attachWithChannelParamsModesAndChannelModes'; + try { + var realtime = helper.AblyRealtime(realtimeOpts); + realtime.connection.on('connected', function () { + var paramsModes = ['presence', 'subscribe']; + var params = { + modes: paramsModes.join(','), + }; + var channelOptions = { + params: params, + modes: ['publish', 'presence_subscribe'], + }; + var channel = realtime.channels.get(testName, channelOptions); + Helper.whenPromiseSettles(channel.attach(), function (err) { + if (err) { + helper.closeAndFinish(done, realtime, err); + return; + } + try { + helper.recordPrivateApi('read.channel.channelOptions'); + expect(channel.channelOptions).to.deep.equal(channelOptions, 'Check requested channel options'); + expect(channel.params).to.deep.equal(params, 'Check result params'); + expect(channel.modes).to.deep.equal(paramsModes, 'Check result modes'); + } catch (err) { + helper.closeAndFinish(done, realtime, err); + return; + } - var testRealtime = helper.AblyRealtime(); - testRealtime.connection.on('connected', function () { - var testChannel = testRealtime.channels.get(testName); - async.series( - [ - checkCanSubscribe(channel, testChannel), - checkCanEnterPresence(channel), - checkCantPublish(channel), - checkCantPresenceSubscribe(channel, testChannel), - ], - function (err) { - testRealtime.close(); - helper.closeAndFinish(done, realtime, err); - }, - ); + var testRealtime = helper.AblyRealtime(); + testRealtime.connection.on('connected', function () { + var testChannel = testRealtime.channels.get(testName); + async.series( + [ + checkCanSubscribe(channel, testChannel), + checkCanEnterPresence(channel), + checkCantPublish(channel), + checkCantPresenceSubscribe(channel, testChannel), + ], + function (err) { + testRealtime.close(); + helper.closeAndFinish(done, realtime, err); + }, + ); + }); }); }); - }); - helper.monitorConnection(done, realtime); - } catch (err) { - helper.closeAndFinish(done, realtime, err); - } - }; - }); + helper.monitorConnection(done, realtime); + } catch (err) { + helper.closeAndFinish(done, realtime, err); + } + }; + }, + ); /** * No spec items found for 'modes' property behavior (like preventing publish, entering presence, presence subscription) @@ -877,7 +881,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL4l * @spec RTL4m */ - Helper.testOnAllTransports(this, 'attachWithChannelModes', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'attachWithChannelModes', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'attachWithChannelModes'; @@ -937,7 +941,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL4l * @spec RTL4m */ - Helper.testOnAllTransports(this, 'attachWithChannelParamsDeltaAndModes', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'attachWithChannelParamsDeltaAndModes', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'attachWithChannelParamsDeltaAndModes'; diff --git a/test/realtime/crypto.test.js b/test/realtime/crypto.test.js index 18df2f11e4..695cff5033 100644 --- a/test/realtime/crypto.test.js +++ b/test/realtime/crypto.test.js @@ -465,7 +465,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async } function single_send(done, helper, realtimeOpts, keyLength) { - // the _128 and _256 variants both call this so it makes more sense for this to be the parameterisedTestTitle instead of that set by testOnAllTransports + // the _128 and _256 variants both call this so it makes more sense for this to be the parameterisedTestTitle instead of that set by testOnAllTransportsAndProtocols helper = helper.withParameterisedTestTitle('single_send'); if (!Crypto) { @@ -513,14 +513,14 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async // Publish and subscribe, various transport, 128 and 256-bit /** @specpartial RSL5b - test aes 128 */ - Helper.testOnAllTransports(this, 'single_send_128', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'single_send_128', function (realtimeOpts) { return function (done) { single_send(done, this.test.helper, realtimeOpts, 128); }; }); /** @specpartial RSL5b - test aes 256 */ - Helper.testOnAllTransports(this, 'single_send_256', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'single_send_256', function (realtimeOpts) { return function (done) { single_send(done, this.test.helper, realtimeOpts, 256); }; diff --git a/test/realtime/failure.test.js b/test/realtime/failure.test.js index 8b59c51e73..3440fbef14 100644 --- a/test/realtime/failure.test.js +++ b/test/realtime/failure.test.js @@ -619,7 +619,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); /** @specpartial RTN14d - last sentence: check that if we received a 5xx disconnected, when we try again we use a fallback host */ - Helper.testOnAllTransports(this, 'try_fallback_hosts_on_placement_constraint', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'try_fallback_hosts_on_placement_constraint', function (realtimeOpts) { return function (done) { const helper = this.test.helper; /* Use the echoserver as a fallback host because it doesn't support diff --git a/test/realtime/message.test.js b/test/realtime/message.test.js index 15389d7f38..100f964699 100644 --- a/test/realtime/message.test.js +++ b/test/realtime/message.test.js @@ -79,7 +79,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * Test publishes in quick succession (on successive ticks of the event loop) * @spec RTL6b */ - Helper.testOnAllTransports(this, 'publishfast', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'publishfast', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -146,7 +146,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL6c2 * @specpartial RTL3d - test processing queued messages */ - Helper.testOnAllTransports(this, 'publishQueued', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'publishQueued', function (realtimeOpts) { return function (done) { var helper = this.test.helper, txRealtime, @@ -629,7 +629,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL6 * @spec RTL6b */ - Helper.testOnAllTransports(this, 'publish', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'publish', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var count = 10; diff --git a/test/realtime/reauth.test.js b/test/realtime/reauth.test.js index 0653bdc3e7..081b3cd2b5 100644 --- a/test/realtime/reauth.test.js +++ b/test/realtime/reauth.test.js @@ -178,7 +178,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { } function testCase(thisInDescribe, name, createSteps) { - Helper.testOnAllTransports(thisInDescribe, name, function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(thisInDescribe, name, function (realtimeOpts) { return function (done) { const helper = this.test.helper; var _steps = createSteps(helper).slice(); diff --git a/test/realtime/resume.test.js b/test/realtime/resume.test.js index 0ae4554dd3..7fd47def5a 100644 --- a/test/realtime/resume.test.js +++ b/test/realtime/resume.test.js @@ -140,7 +140,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * Related to RTN15b, RTN15c. * @nospec */ - Helper.testOnAllTransports(this, 'resume_inactive', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'resume_inactive', function (realtimeOpts) { return function (done) { resume_inactive(done, this.test.helper, 'resume_inactive' + String(Math.random()), {}, realtimeOpts); }; @@ -266,7 +266,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * Related to RTN15b, RTN15c. * @nospec */ - Helper.testOnAllTransports(this, 'resume_active', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'resume_active', function (realtimeOpts) { return function (done) { resume_active(done, this.test.helper, 'resume_active' + String(Math.random()), {}, realtimeOpts); }; @@ -276,7 +276,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * Resume with loss of continuity * @spec RTN15c7 */ - Helper.testOnAllTransports( + Helper.testOnAllTransportsAndProtocols( this, 'resume_lost_continuity', function (realtimeOpts) { @@ -353,7 +353,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * Resume with token error * @spec RTN15c5 */ - Helper.testOnAllTransports( + Helper.testOnAllTransportsAndProtocols( this, 'resume_token_error', function (realtimeOpts) { @@ -411,7 +411,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * Resume with fatal error * @spec RTN15c4 */ - Helper.testOnAllTransports( + Helper.testOnAllTransportsAndProtocols( this, 'resume_fatal_error', function (realtimeOpts) { From 9be2e539d1345eaec958d153a0e988a7718ea503 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Fri, 31 Jan 2025 05:09:57 +0000 Subject: [PATCH 126/166] Run LiveObjects tests on all transports and protocols (MsgPack and JSON) Only the tests that interact with the realtime server are marked to run using `SharedHelper.testOnAllTransportsAndProtocols` function. Tests that rely on private `channel.processMessage` calls to inject forged state messages are not included. Resolves DTP-1096 --- test/realtime/live_objects.test.js | 422 +++++++++++++++++------------ 1 file changed, 248 insertions(+), 174 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 9cfa0ea887..52344122d6 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -34,14 +34,29 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(object.constructor.name).to.match(new RegExp(`_?${className}`), msg); } - function forScenarios(scenarios, testFn) { + function forScenarios(thisInDescribe, scenarios, testFn) { for (const scenario of scenarios) { - const itFn = scenario.skip ? it.skip : scenario.only ? it.only : it; + if (scenario.allTransportsAndProtocols) { + Helper.testOnAllTransportsAndProtocols( + thisInDescribe, + scenario.description, + function (options, channelName) { + return async function () { + const helper = this.test.helper; + await testFn(helper, scenario, options, channelName); + }; + }, + scenario.skip, + scenario.only, + ); + } else { + const itFn = scenario.skip ? it.skip : scenario.only ? it.only : it; - itFn(scenario.description, async function () { - const helper = this.test.helper; - await testFn(helper, scenario); - }); + itFn(scenario.description, async function () { + const helper = this.test.helper; + await testFn(helper, scenario, {}, scenario.description); + }); + } } } @@ -404,175 +419,209 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); /** @nospec */ - it('builds state object tree from STATE_SYNC sequence on channel attachment', async function () { - const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper); - - await helper.monitorConnectionThenCloseAndFinish(async () => { - await waitFixtureChannelIsReady(client); - - const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; - - await channel.attach(); - const root = await liveObjects.getRoot(); - - const counterKeys = ['emptyCounter', 'initialValueCounter', 'referencedCounter']; - const mapKeys = ['emptyMap', 'referencedMap', 'valuesMap']; - const rootKeysCount = counterKeys.length + mapKeys.length; - - expect(root, 'Check getRoot() is resolved when STATE_SYNC sequence ends').to.exist; - expect(root.size()).to.equal(rootKeysCount, 'Check root has correct number of keys'); - - counterKeys.forEach((key) => { - const counter = root.get(key); - expect(counter, `Check counter at key="${key}" in root exists`).to.exist; - expectInstanceOf(counter, 'LiveCounter', `Check counter at key="${key}" in root is of type LiveCounter`); - }); + Helper.testOnAllTransportsAndProtocols( + this, + 'builds state object tree from STATE_SYNC sequence on channel attachment', + function (options, channelName) { + return async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper, options); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + await waitFixtureChannelIsReady(client); + + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + const counterKeys = ['emptyCounter', 'initialValueCounter', 'referencedCounter']; + const mapKeys = ['emptyMap', 'referencedMap', 'valuesMap']; + const rootKeysCount = counterKeys.length + mapKeys.length; + + expect(root, 'Check getRoot() is resolved when STATE_SYNC sequence ends').to.exist; + expect(root.size()).to.equal(rootKeysCount, 'Check root has correct number of keys'); + + counterKeys.forEach((key) => { + const counter = root.get(key); + expect(counter, `Check counter at key="${key}" in root exists`).to.exist; + expectInstanceOf( + counter, + 'LiveCounter', + `Check counter at key="${key}" in root is of type LiveCounter`, + ); + }); - mapKeys.forEach((key) => { - const map = root.get(key); - expect(map, `Check map at key="${key}" in root exists`).to.exist; - expectInstanceOf(map, 'LiveMap', `Check map at key="${key}" in root is of type LiveMap`); - }); + mapKeys.forEach((key) => { + const map = root.get(key); + expect(map, `Check map at key="${key}" in root exists`).to.exist; + expectInstanceOf(map, 'LiveMap', `Check map at key="${key}" in root is of type LiveMap`); + }); - const valuesMap = root.get('valuesMap'); - const valueMapKeys = [ - 'stringKey', - 'emptyStringKey', - 'bytesKey', - 'emptyBytesKey', - 'numberKey', - 'zeroKey', - 'trueKey', - 'falseKey', - 'mapKey', - ]; - expect(valuesMap.size()).to.equal(valueMapKeys.length, 'Check nested map has correct number of keys'); - valueMapKeys.forEach((key) => { - const value = valuesMap.get(key); - expect(value, `Check value at key="${key}" in nested map exists`).to.exist; - }); - }, client); - }); + const valuesMap = root.get('valuesMap'); + const valueMapKeys = [ + 'stringKey', + 'emptyStringKey', + 'bytesKey', + 'emptyBytesKey', + 'numberKey', + 'zeroKey', + 'trueKey', + 'falseKey', + 'mapKey', + ]; + expect(valuesMap.size()).to.equal(valueMapKeys.length, 'Check nested map has correct number of keys'); + valueMapKeys.forEach((key) => { + const value = valuesMap.get(key); + expect(value, `Check value at key="${key}" in nested map exists`).to.exist; + }); + }, client); + }; + }, + ); /** @nospec */ - it('LiveCounter is initialized with initial value from STATE_SYNC sequence', async function () { - const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper); - - await helper.monitorConnectionThenCloseAndFinish(async () => { - await waitFixtureChannelIsReady(client); - - const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; - - await channel.attach(); - const root = await liveObjects.getRoot(); - - const counters = [ - { key: 'emptyCounter', value: 0 }, - { key: 'initialValueCounter', value: 10 }, - { key: 'referencedCounter', value: 20 }, - ]; - - counters.forEach((x) => { - const counter = root.get(x.key); - expect(counter.value()).to.equal(x.value, `Check counter at key="${x.key}" in root has correct value`); - }); - }, client); - }); + Helper.testOnAllTransportsAndProtocols( + this, + 'LiveCounter is initialized with initial value from STATE_SYNC sequence', + function (options, channelName) { + return async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper, options); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + await waitFixtureChannelIsReady(client); + + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + const counters = [ + { key: 'emptyCounter', value: 0 }, + { key: 'initialValueCounter', value: 10 }, + { key: 'referencedCounter', value: 20 }, + ]; + + counters.forEach((x) => { + const counter = root.get(x.key); + expect(counter.value()).to.equal(x.value, `Check counter at key="${x.key}" in root has correct value`); + }); + }, client); + }; + }, + ); /** @nospec */ - it('LiveMap is initialized with initial value from STATE_SYNC sequence', async function () { - const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper); + Helper.testOnAllTransportsAndProtocols( + this, + 'LiveMap is initialized with initial value from STATE_SYNC sequence', + function (options, channelName) { + return async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper, options); - await helper.monitorConnectionThenCloseAndFinish(async () => { - await waitFixtureChannelIsReady(client); + await helper.monitorConnectionThenCloseAndFinish(async () => { + await waitFixtureChannelIsReady(client); - const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; - await channel.attach(); - const root = await liveObjects.getRoot(); + await channel.attach(); + const root = await liveObjects.getRoot(); - const emptyMap = root.get('emptyMap'); - expect(emptyMap.size()).to.equal(0, 'Check empty map in root has no keys'); + const emptyMap = root.get('emptyMap'); + expect(emptyMap.size()).to.equal(0, 'Check empty map in root has no keys'); - const referencedMap = root.get('referencedMap'); - expect(referencedMap.size()).to.equal(1, 'Check referenced map in root has correct number of keys'); + const referencedMap = root.get('referencedMap'); + expect(referencedMap.size()).to.equal(1, 'Check referenced map in root has correct number of keys'); - const counterFromReferencedMap = referencedMap.get('counterKey'); - expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); + const counterFromReferencedMap = referencedMap.get('counterKey'); + expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); - const valuesMap = root.get('valuesMap'); - expect(valuesMap.size()).to.equal(9, 'Check values map in root has correct number of keys'); + const valuesMap = root.get('valuesMap'); + expect(valuesMap.size()).to.equal(9, 'Check values map in root has correct number of keys'); - expect(valuesMap.get('stringKey')).to.equal('stringValue', 'Check values map has correct string value key'); - expect(valuesMap.get('emptyStringKey')).to.equal('', 'Check values map has correct empty string value key'); - helper.recordPrivateApi('call.BufferUtils.base64Decode'); - helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); - expect( - BufferUtils.areBuffersEqual( - valuesMap.get('bytesKey'), - BufferUtils.base64Decode('eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9'), - ), - 'Check values map has correct bytes value key', - ).to.be.true; - helper.recordPrivateApi('call.BufferUtils.base64Decode'); - helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); - expect( - BufferUtils.areBuffersEqual(valuesMap.get('emptyBytesKey'), BufferUtils.base64Decode('')), - 'Check values map has correct empty bytes value key', - ).to.be.true; - expect(valuesMap.get('numberKey')).to.equal(1, 'Check values map has correct number value key'); - expect(valuesMap.get('zeroKey')).to.equal(0, 'Check values map has correct zero number value key'); - expect(valuesMap.get('trueKey')).to.equal(true, `Check values map has correct 'true' value key`); - expect(valuesMap.get('falseKey')).to.equal(false, `Check values map has correct 'false' value key`); - - const mapFromValuesMap = valuesMap.get('mapKey'); - expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); - }, client); - }); + expect(valuesMap.get('stringKey')).to.equal( + 'stringValue', + 'Check values map has correct string value key', + ); + expect(valuesMap.get('emptyStringKey')).to.equal( + '', + 'Check values map has correct empty string value key', + ); + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual( + valuesMap.get('bytesKey'), + BufferUtils.base64Decode('eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9'), + ), + 'Check values map has correct bytes value key', + ).to.be.true; + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual(valuesMap.get('emptyBytesKey'), BufferUtils.base64Decode('')), + 'Check values map has correct empty bytes value key', + ).to.be.true; + expect(valuesMap.get('numberKey')).to.equal(1, 'Check values map has correct number value key'); + expect(valuesMap.get('zeroKey')).to.equal(0, 'Check values map has correct zero number value key'); + expect(valuesMap.get('trueKey')).to.equal(true, `Check values map has correct 'true' value key`); + expect(valuesMap.get('falseKey')).to.equal(false, `Check values map has correct 'false' value key`); + + const mapFromValuesMap = valuesMap.get('mapKey'); + expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); + }, client); + }; + }, + ); /** @nospec */ - it('LiveMaps can reference the same object in their keys', async function () { - const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper); - - await helper.monitorConnectionThenCloseAndFinish(async () => { - await waitFixtureChannelIsReady(client); - - const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; - - await channel.attach(); - const root = await liveObjects.getRoot(); - - const referencedCounter = root.get('referencedCounter'); - const referencedMap = root.get('referencedMap'); - const valuesMap = root.get('valuesMap'); - - const counterFromReferencedMap = referencedMap.get('counterKey'); - expect(counterFromReferencedMap, 'Check nested counter exists at a key in a map').to.exist; - expectInstanceOf(counterFromReferencedMap, 'LiveCounter', 'Check nested counter is of type LiveCounter'); - expect(counterFromReferencedMap).to.equal( - referencedCounter, - 'Check nested counter is the same object instance as counter on the root', - ); - expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); - - const mapFromValuesMap = valuesMap.get('mapKey'); - expect(mapFromValuesMap, 'Check nested map exists at a key in a map').to.exist; - expectInstanceOf(mapFromValuesMap, 'LiveMap', 'Check nested map is of type LiveMap'); - expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); - expect(mapFromValuesMap).to.equal( - referencedMap, - 'Check nested map is the same object instance as map on the root', - ); - }, client); - }); + Helper.testOnAllTransportsAndProtocols( + this, + 'LiveMap can reference the same object in their keys', + function (options, channelName) { + return async function () { + const helper = this.test.helper; + const client = RealtimeWithLiveObjects(helper, options); + + await helper.monitorConnectionThenCloseAndFinish(async () => { + await waitFixtureChannelIsReady(client); + + const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); + const liveObjects = channel.liveObjects; + + await channel.attach(); + const root = await liveObjects.getRoot(); + + const referencedCounter = root.get('referencedCounter'); + const referencedMap = root.get('referencedMap'); + const valuesMap = root.get('valuesMap'); + + const counterFromReferencedMap = referencedMap.get('counterKey'); + expect(counterFromReferencedMap, 'Check nested counter exists at a key in a map').to.exist; + expectInstanceOf(counterFromReferencedMap, 'LiveCounter', 'Check nested counter is of type LiveCounter'); + expect(counterFromReferencedMap).to.equal( + referencedCounter, + 'Check nested counter is the same object instance as counter on the root', + ); + expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); + + const mapFromValuesMap = valuesMap.get('mapKey'); + expect(mapFromValuesMap, 'Check nested map exists at a key in a map').to.exist; + expectInstanceOf(mapFromValuesMap, 'LiveMap', 'Check nested map is of type LiveMap'); + expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); + expect(mapFromValuesMap).to.equal( + referencedMap, + 'Check nested map is the same object instance as map on the root', + ); + }, client); + }; + }, + ); const primitiveKeyData = [ { key: 'stringKey', data: { value: 'stringValue' } }, @@ -663,6 +712,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'STATE_SYNC sequence with state object "tombstone" property deletes existing object', action: async (ctx) => { const { root, liveObjectsHelper, channelName, channel } = ctx; @@ -712,6 +762,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'STATE_SYNC sequence with state object "tombstone" property triggers subscription callback for existing object', action: async (ctx) => { @@ -769,6 +820,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const applyOperationsScenarios = [ { + allTransportsAndProtocols: true, description: 'can apply MAP_CREATE with primitives state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName, helper } = ctx; @@ -832,6 +884,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'can apply MAP_CREATE with object ids state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -989,6 +1042,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'can apply MAP_SET with primitives state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName, helper } = ctx; @@ -1037,6 +1091,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'can apply MAP_SET with object ids state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -1163,6 +1218,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'can apply MAP_REMOVE state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -1297,6 +1353,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'can apply COUNTER_CREATE state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -1419,6 +1476,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'can apply COUNTER_INC state operation messages', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -2217,6 +2275,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const writeApiScenarios = [ { + allTransportsAndProtocols: true, description: 'LiveCounter.increment sends COUNTER_INC operation', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -2326,6 +2385,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'LiveCounter.decrement sends COUNTER_INC operation', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -2435,6 +2495,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'LiveMap.set sends MAP_SET operation with primitive values', action: async (ctx) => { const { root, helper } = ctx; @@ -2469,6 +2530,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'LiveMap.set sends MAP_SET operation with reference to another LiveObject', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -2546,6 +2608,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'LiveMap.remove sends MAP_REMOVE operation', action: async (ctx) => { const { root, liveObjectsHelper, channelName } = ctx; @@ -2608,6 +2671,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'LiveObjects.createCounter sends COUNTER_CREATE operation', action: async (ctx) => { const { liveObjects } = ctx; @@ -2629,6 +2693,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'LiveCounter created with LiveObjects.createCounter can be assigned to the state tree', action: async (ctx) => { const { root, liveObjects } = ctx; @@ -2671,6 +2736,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'LiveObjects.createCounter can return LiveCounter with initial value from applied CREATE operation', action: async (ctx) => { @@ -2702,7 +2768,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: - 'Initial value is not double counted for LiveCounter from LiveObjects.createCounter when CREATE op is received', + 'initial value is not double counted for LiveCounter from LiveObjects.createCounter when CREATE op is received', action: async (ctx) => { const { liveObjects, liveObjectsHelper, helper, channel } = ctx; @@ -2783,6 +2849,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'LiveObjects.createMap sends MAP_CREATE operation with primitive values', action: async (ctx) => { const { liveObjects, helper } = ctx; @@ -2836,6 +2903,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'LiveObjects.createMap sends MAP_CREATE operation with reference to another LiveObject', action: async (ctx) => { const { root, liveObjectsHelper, channelName, liveObjects } = ctx; @@ -2876,6 +2944,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'LiveMap created with LiveObjects.createMap can be assigned to the state tree', action: async (ctx) => { const { root, liveObjects } = ctx; @@ -2919,6 +2988,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'LiveObjects.createMap can return LiveMap with initial value from applied CREATE operation', action: async (ctx) => { const { liveObjects, liveObjectsHelper, helper, channel } = ctx; @@ -2955,7 +3025,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: - 'Initial value is not double counted for LiveMap from LiveObjects.createMap when CREATE op is received', + 'initial value is not double counted for LiveMap from LiveObjects.createMap when CREATE op is received', action: async (ctx) => { const { liveObjects, liveObjectsHelper, helper, channel } = ctx; @@ -3181,6 +3251,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'batch API scheduled operations are applied when batch callback is finished', action: async (ctx) => { const { root, liveObjects } = ctx; @@ -3361,18 +3432,18 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ forScenarios( + this, [ ...stateSyncSequenceScenarios, ...applyOperationsScenarios, ...applyOperationsDuringSyncScenarios, ...writeApiScenarios, ], - async function (helper, scenario) { + async function (helper, scenario, clientOptions, channelName) { const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper); + const client = RealtimeWithLiveObjects(helper, clientOptions); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channelName = scenario.description; const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); const liveObjects = channel.liveObjects; @@ -3386,6 +3457,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const subscriptionCallbacksScenarios = [ { + allTransportsAndProtocols: true, description: 'can subscribe to the incoming COUNTER_INC operation on a LiveCounter', action: async (ctx) => { const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; @@ -3418,6 +3490,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'can subscribe to multiple incoming operations on a LiveCounter', action: async (ctx) => { const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; @@ -3461,6 +3534,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'can subscribe to the incoming MAP_SET operation on a LiveMap', action: async (ctx) => { const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; @@ -3494,6 +3568,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'can subscribe to the incoming MAP_REMOVE operation on a LiveMap', action: async (ctx) => { const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; @@ -3526,6 +3601,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'can subscribe to multiple incoming operations on a LiveMap', action: async (ctx) => { const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; @@ -3864,12 +3940,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ]; /** @nospec */ - forScenarios(subscriptionCallbacksScenarios, async function (helper, scenario) { + forScenarios(this, subscriptionCallbacksScenarios, async function (helper, scenario, clientOptions, channelName) { const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper); + const client = RealtimeWithLiveObjects(helper, clientOptions); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channelName = scenario.description; const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); const liveObjects = channel.liveObjects; @@ -3965,6 +4040,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { + allTransportsAndProtocols: true, description: 'tombstoned map entry is removed from the LiveMap after the GC grace period', action: async (ctx) => { const { root, liveObjectsHelper, channelName, helper, waitForGCCycles } = ctx; @@ -4014,7 +4090,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ]; /** @nospec */ - forScenarios(tombstonesGCScenarios, async function (helper, scenario) { + forScenarios(this, tombstonesGCScenarios, async function (helper, scenario, clientOptions, channelName) { try { helper.recordPrivateApi('write.LiveObjects._DEFAULTS.gcInterval'); LiveObjectsPlugin.LiveObjects._DEFAULTS.gcInterval = 500; @@ -4022,10 +4098,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], LiveObjectsPlugin.LiveObjects._DEFAULTS.gcGracePeriod = 250; const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper); + const client = RealtimeWithLiveObjects(helper, clientOptions); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channelName = scenario.description; const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); const liveObjects = channel.liveObjects; @@ -4179,12 +4254,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ]; /** @nospec */ - forScenarios(missingChannelModesScenarios, async function (helper, scenario) { + forScenarios(this, missingChannelModesScenarios, async function (helper, scenario, clientOptions, channelName) { const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper); + const client = RealtimeWithLiveObjects(helper, clientOptions); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channelName = scenario.description; // attach with correct channel modes so we can create liveobjects on the root for testing. // each scenario will modify the underlying modes array to test specific behavior const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); @@ -4467,7 +4541,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ]; /** @nospec */ - forScenarios(stateMessageSizeScenarios, function (helper, scenario) { + forScenarios(this, stateMessageSizeScenarios, function (helper, scenario) { helper.recordPrivateApi('call.StateMessage.encode'); LiveObjectsPlugin.StateMessage.encode(scenario.message); helper.recordPrivateApi('call.BufferUtils.utf8Encode'); // was called by a scenario to create buffers From bf842fa643a667063717e58c819a15b61841832e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 12 Feb 2025 06:14:40 +0000 Subject: [PATCH 127/166] Use more specific error codes for LiveObjects errors Resolves PUB-1011 --- .../liveobjects/batchcontextlivecounter.ts | 2 +- src/plugins/liveobjects/livecounter.ts | 18 ++++++------- src/plugins/liveobjects/livemap.ts | 26 +++++++++---------- src/plugins/liveobjects/liveobject.ts | 4 +-- src/plugins/liveobjects/liveobjects.ts | 4 +-- src/plugins/liveobjects/objectid.ts | 10 +++---- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/plugins/liveobjects/batchcontextlivecounter.ts b/src/plugins/liveobjects/batchcontextlivecounter.ts index e61cd9d1c3..3528c3ecbd 100644 --- a/src/plugins/liveobjects/batchcontextlivecounter.ts +++ b/src/plugins/liveobjects/batchcontextlivecounter.ts @@ -33,7 +33,7 @@ export class BatchContextLiveCounter { // do an explicit type safety check here before negating the amount value, // so we don't unintentionally change the type sent by a user if (typeof amount !== 'number') { - throw new this._client.ErrorInfo('Counter value decrement should be a number', 40013, 400); + throw new this._client.ErrorInfo('Counter value decrement should be a number', 40003, 400); } this.increment(-amount); diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index c5a7be1bbe..34b9ecf594 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -52,7 +52,7 @@ export class LiveCounter extends LiveObject const client = liveObjects.getClient(); if (typeof amount !== 'number' || !isFinite(amount)) { - throw new client.ErrorInfo('Counter value increment should be a valid number', 40013, 400); + throw new client.ErrorInfo('Counter value increment should be a valid number', 40003, 400); } const stateMessage = StateMessage.fromValues( @@ -77,7 +77,7 @@ export class LiveCounter extends LiveObject const client = liveObjects.getClient(); if (count !== undefined && (typeof count !== 'number' || !Number.isFinite(count))) { - throw new client.ErrorInfo('Counter value should be a valid number', 40013, 400); + throw new client.ErrorInfo('Counter value should be a valid number', 40003, 400); } const initialValueObj = LiveCounter.createInitialValueObject(count); @@ -150,7 +150,7 @@ export class LiveCounter extends LiveObject // do an explicit type safety check here before negating the amount value, // so we don't unintentionally change the type sent by a user if (typeof amount !== 'number' || !isFinite(amount)) { - throw new this._client.ErrorInfo('Counter value decrement should be a valid number', 40013, 400); + throw new this._client.ErrorInfo('Counter value decrement should be a valid number', 40003, 400); } return this.increment(-amount); @@ -163,7 +163,7 @@ export class LiveCounter extends LiveObject if (op.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( `Cannot apply state operation with objectId=${op.objectId}, to this LiveCounter with objectId=${this.getObjectId()}`, - 50000, + 92000, 500, ); } @@ -211,7 +211,7 @@ export class LiveCounter extends LiveObject default: throw new this._client.ErrorInfo( `Invalid ${op.action} op for LiveCounter objectId=${this.getObjectId()}`, - 50000, + 92000, 500, ); } @@ -226,7 +226,7 @@ export class LiveCounter extends LiveObject if (stateObject.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( `Invalid state object: state object objectId=${stateObject.objectId}; LiveCounter objectId=${this.getObjectId()}`, - 50000, + 92000, 500, ); } @@ -236,7 +236,7 @@ export class LiveCounter extends LiveObject if (stateObject.createOp.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( `Invalid state object: state object createOp objectId=${stateObject.createOp?.objectId}; LiveCounter objectId=${this.getObjectId()}`, - 50000, + 92000, 500, ); } @@ -244,7 +244,7 @@ export class LiveCounter extends LiveObject if (stateObject.createOp.action !== StateOperationAction.COUNTER_CREATE) { throw new this._client.ErrorInfo( `Invalid state object: state object createOp action=${stateObject.createOp?.action}; LiveCounter objectId=${this.getObjectId()}`, - 50000, + 92000, 500, ); } @@ -308,7 +308,7 @@ export class LiveCounter extends LiveObject private _throwNoPayloadError(op: StateOperation): void { throw new this._client.ErrorInfo( `No payload found for ${op.action} op for LiveCounter objectId=${this.getObjectId()}`, - 50000, + 92000, 500, ); } diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 6b9f60e80e..637c5eeed0 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -143,7 +143,7 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject LiveMap.validateKeyValue(liveObjects, key, value)); @@ -368,7 +368,7 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject Date: Wed, 12 Feb 2025 07:57:10 +0000 Subject: [PATCH 128/166] Fix LiveObjects docstrings --- src/plugins/liveobjects/livecounter.ts | 2 +- src/plugins/liveobjects/liveobjects.ts | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index c5a7be1bbe..9871d235e1 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -143,7 +143,7 @@ export class LiveCounter extends LiveObject } /** - * Alias for calling {@link LiveCounter.increment | LiveCounter.increment(-amount)} + * An alias for calling {@link LiveCounter.increment | LiveCounter.increment(-amount)} */ async decrement(amount: number): Promise { this._liveObjects.throwIfMissingStatePublishMode(); diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 7458e3d640..181514e818 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -101,11 +101,10 @@ export class LiveObjects { /** * Send a MAP_CREATE operation to the realtime system to create a new map object in the pool. * - * Locally on the client it creates a zero-value object with the corresponding id and returns it. - * The object initialization with the initial value is expected to happen when the corresponding MAP_CREATE operation is echoed - * back to the client and applied to the object following the regular operation application procedure. + * Once the ACK message is received, the method returns the object from the local pool if it got created due to + * the echoed MAP_CREATE operation, or if it wasn't received yet, the method creates a new object locally using the provided data and returns it. * - * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with a zero-value object created in the local pool. + * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with an object containing provided data. */ async createMap(entries?: T): Promise> { this.throwIfMissingStatePublishMode(); @@ -134,11 +133,10 @@ export class LiveObjects { /** * Send a COUNTER_CREATE operation to the realtime system to create a new counter object in the pool. * - * Locally on the client it creates a zero-value object with the corresponding id and returns it. - * The object initialization with the initial value is expected to happen when the corresponding COUNTER_CREATE operation is echoed - * back to the client and applied to the object following the regular operation application procedure. + * Once the ACK message is received, the method returns the object from the local pool if it got created due to + * the echoed COUNTER_CREATE operation, or if it wasn't received yet, the method creates a new object locally using the provided data and returns it. * - * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with a zero-value object created in the local pool. + * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with an object containing provided data. */ async createCounter(count?: number): Promise { this.throwIfMissingStatePublishMode(); From f2dcf8a08c30c3bb4d6ec6dc7560f6a82648feac Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 12 Feb 2025 08:21:36 +0000 Subject: [PATCH 129/166] Add type declarations to `ably.d.ts` for Live Objects write, batch and lifecycle events APIs --- ably.d.ts | 274 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) diff --git a/ably.d.ts b/ably.d.ts index eaccb3ccb6..4ed3c0811c 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -1566,6 +1566,25 @@ export type ErrorCallback = (error: ErrorInfo | null) => void; */ export type LiveObjectUpdateCallback = (update: T) => void; +/** + * The callback used for the events emitted by {@link LiveObjects}. + */ +export type LiveObjectsEventCallback = () => void; + +/** + * The callback used for the lifecycle events emitted by {@link LiveObject}. + */ +export type LiveObjectLifecycleEventCallback = () => void; + +/** + * A function passed to {@link LiveObjects.batch} to group multiple Live Object operations into a single channel message. + * + * Must not be `async`. + * + * @param batchContext - A {@link BatchContext} object that allows grouping operations on Live Objects for this batch. + */ +export type BatchCallback = (batchContext: BatchContext) => void; + // Internal Interfaces // To allow a uniform (callback) interface between on and once even in the @@ -2032,6 +2051,40 @@ export declare interface PushChannel { listSubscriptions(params?: Record): Promise>; } +/** + * The `LiveObjectsEvents` namespace describes the possible values of the {@link LiveObjectsEvent} type. + */ +declare namespace LiveObjectsEvents { + /** + * The local Live Objects state is currently being synchronized with the Ably service. + */ + type SYNCING = 'syncing'; + /** + * The local Live Objects state has been synchronized with the Ably service. + */ + type SYNCED = 'synced'; +} + +/** + * Describes the events emitted by a {@link LiveObjects} object. + */ +export type LiveObjectsEvent = LiveObjectsEvents.SYNCED | LiveObjectsEvents.SYNCING; + +/** + * The `LiveObjectLifecycleEvents` namespace describes the possible values of the {@link LiveObjectLifecycleEvent} type. + */ +declare namespace LiveObjectLifecycleEvents { + /** + * Indicates that the object has been deleted from the Live Objects pool and should no longer be interacted with. + */ + type DELETED = 'deleted'; +} + +/** + * Describes the events emitted by a {@link LiveObject} object. + */ +export type LiveObjectLifecycleEvent = LiveObjectLifecycleEvents.DELETED; + /** * Enables the LiveObjects state to be subscribed to for a channel. */ @@ -2062,6 +2115,59 @@ export declare interface LiveObjects { * @returns A promise which, upon success, will be fulfilled with a {@link LiveMap} object. Upon failure, the promise will be rejected with an {@link ErrorInfo} object which explains the error. */ getRoot(): Promise>; + + /** + * Creates a new {@link LiveMap} object instance with the provided entries. + * + * @param entries - The initial entries for the new {@link LiveMap} object. + * @returns A promise which, upon success, will be fulfilled with a {@link LiveMap} object. Upon failure, the promise will be rejected with an {@link ErrorInfo} object which explains the error. + */ + createMap(entries?: T): Promise>; + + /** + * Creates a new {@link LiveCounter} object instance with the provided `count` value. + * + * @param count - The initial value for the new {@link LiveCounter} object. + * @returns A promise which, upon success, will be fulfilled with a {@link LiveCounter} object. Upon failure, the promise will be rejected with an {@link ErrorInfo} object which explains the error. + */ + createCounter(count?: number): Promise; + + /** + * Allows you to group multiple operations together and send them to the Ably service in a single channel message. + * As a result, other clients will receive the changes as a single channel message after the batch function has completed. + * + * This method accepts a synchronous callback, which is provided with a {@link BatchContext} object. + * Use the context object to access Live Object instances in your state and batch operations for them. + * + * The objects' data is not modified inside the callback function. Instead, the objects will be updated + * when the batched operations are applied by the Ably service and echoed back to the client. + * + * @param callback - A batch callback function used to group operations together. Cannot be an `async` function. + * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + */ + batch(callback: BatchCallback): Promise; + + /** + * Registers the provided listener for the specified event. If `on()` is called more than once with the same listener and event, the listener is added multiple times to its listener registry. Therefore, as an example, assuming the same listener is registered twice using `on()`, and an event is emitted once, the listener would be invoked twice. + * + * @param event - The named event to listen for. + * @param callback - The event listener. + * @returns A {@link OnLiveObjectsEventResponse} object that allows the provided listener to be deregistered from future updates. + */ + on(event: LiveObjectsEvent, callback: LiveObjectsEventCallback): OnLiveObjectsEventResponse; + + /** + * Removes all registrations that match both the specified listener and the specified event. + * + * @param event - The named event. + * @param callback - The event listener. + */ + off(event: LiveObjectsEvent, callback: LiveObjectsEventCallback): void; + + /** + * Deregisters all registrations, for all events and listeners. + */ + offAll(): void; } declare global { @@ -2095,6 +2201,97 @@ export type DefaultRoot = ? LiveObjectsTypes['root'] // "root" property exists, and it is of an expected type, we can use this interface for the root object in LiveObjects. : `Provided type definition for the "root" object in LiveObjectsTypes is not of an expected LiveMapType`; +/** + * Object returned from an `on` call, allowing the listener provided in that call to be deregistered. + */ +export declare interface OnLiveObjectsEventResponse { + /** + * Deregisters the listener passed to the `on` call. + */ + off(): void; +} + +/** + * Enables grouping multiple Live Object operations together by providing `BatchContext*` wrapper objects. + */ +export declare interface BatchContext { + /** + * Mirrors the {@link LiveObjects.getRoot} method and returns a {@link BatchContextLiveMap} wrapper for the root object on a channel. + * + * @returns A {@link BatchContextLiveMap} object. + */ + getRoot(): BatchContextLiveMap; +} + +/** + * A wrapper around the {@link LiveMap} object that enables batching operations inside a {@link BatchCallback}. + */ +export declare interface BatchContextLiveMap { + /** + * Mirrors the {@link LiveMap.get} method and returns the value associated with a key in the map. + * + * @param key - The key to retrieve the value for. + * @returns A {@link LiveObject}, a primitive type (string, number, boolean, or binary data) or `undefined` if the key doesn't exist in a map or the associated {@link LiveObject} has been deleted. Always `undefined` if this map object is deleted. + */ + get(key: TKey): T[TKey] | undefined; + + /** + * Returns the number of key/value pairs in the map. + */ + size(): number; + + /** + * Similar to the {@link LiveMap.set} method, but instead, it adds an operation to set a key in the map with the provided value to the current batch, to be sent in a single message to the Ably service. + * + * This does not modify the underlying data of this object. Instead, the change is applied when + * the published operation is echoed back to the client and applied to the object. + * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. + * + * @param key - The key to set the value for. + * @param value - The value to assign to the key. + */ + set(key: TKey, value: T[TKey]): void; + + /** + * Similar to the {@link LiveMap.remove} method, but instead, it adds an operation to remove a key from the map to the current batch, to be sent in a single message to the Ably service. + * + * This does not modify the underlying data of this object. Instead, the change is applied when + * the published operation is echoed back to the client and applied to the object. + * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. + * + * @param key - The key to set the value for. + */ + remove(key: TKey): void; +} + +/** + * A wrapper around the {@link LiveCounter} object that enables batching operations inside a {@link BatchCallback}. + */ +export declare interface BatchContextLiveCounter { + /** + * Returns the current value of the counter. + */ + value(): number; + + /** + * Similar to the {@link LiveCounter.increment} method, but instead, it adds an operation to increment the counter value to the current batch, to be sent in a single message to the Ably service. + * + * This does not modify the underlying data of this object. Instead, the change is applied when + * the published operation is echoed back to the client and applied to the object. + * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. + * + * @param amount - The amount by which to increase the counter value. + */ + increment(amount: number): void; + + /** + * An alias for calling {@link BatchContextLiveCounter.increment | BatchContextLiveCounter.increment(-amount)} + * + * @param amount - The amount by which to decrease the counter value. + */ + decrement(amount: number): void; +} + /** * The `LiveMap` class represents a key/value map data structure, similar to a JavaScript Map, where all changes are synchronized across clients in realtime. * Conflict-free resolution for updates follows Last Write Wins (LWW) semantics, meaning that if two clients update the same key in the map, the update with the most recent timestamp wins. @@ -2116,6 +2313,31 @@ export declare interface LiveMap extends LiveObject(key: TKey, value: T[TKey]): Promise; + + /** + * Sends an operation to the Ably system to remove a key from this `LiveMap` object. + * + * This does not modify the underlying data of this object. Instead, the change is applied when + * the published operation is echoed back to the client and applied to the object. + * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. + * + * @param key - The key to remove. + * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + */ + remove(key: TKey): Promise; } /** @@ -2145,6 +2367,26 @@ export declare interface LiveCounter extends LiveObject { * Returns the current value of the counter. */ value(): number; + + /** + * Sends an operation to the Ably system to increment the value of this `LiveCounter` object. + * + * This does not modify the underlying data of this object. Instead, the change is applied when + * the published operation is echoed back to the client and applied to the object. + * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. + * + * @param amount - The amount by which to increase the counter value. + * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + */ + increment(amount: number): Promise; + + /** + * An alias for calling {@link LiveCounter.increment | LiveCounter.increment(-amount)} + * + * @param amount - The amount by which to decrease the counter value. + * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + */ + decrement(amount: number): Promise; } /** @@ -2185,6 +2427,28 @@ export declare interface LiveObject Date: Thu, 13 Feb 2025 03:18:49 +0000 Subject: [PATCH 130/166] Update Live Objects README section Describe the Live Objects create/edit/read API, live object types, root object, state channel modes, batch operations lifecycle events and user provided typings for Live Objects. --- README.md | 292 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- ably.d.ts | 6 +- 2 files changed, 289 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f69d00fd65..4124d939de 100644 --- a/README.md +++ b/README.md @@ -584,13 +584,15 @@ const client = new Ably.Rest({ The Push plugin is developed as part of the Ably client library, so it is available for the same versions as the Ably client library itself. It also means that it follows the same semantic versioning rules as they were defined for [the Ably client library](#for-browsers). For example, to lock into a major or minor version of the Push plugin, you can specify a specific version number such as https://cdn.ably.com/lib/push.umd.min-2.js for all v2._ versions, or https://cdn.ably.com/lib/push.umd.min-2.4.js for all v2.4._ versions, or you can lock into a single release with https://cdn.ably.com/lib/push.umd.min-2.4.0.js. Note you can load the non-minified version by omitting `.min` from the URL such as https://cdn.ably.com/lib/push.umd-2.js. -For more information on publishing push notifcations over Ably, see the [Ably push documentation](https://ably.com/docs/push). +For more information on publishing push notifications over Ably, see the [Ably push documentation](https://ably.com/docs/push). -### Live Objects functionality +### LiveObjects -Live Objects functionality is supported for Realtime clients via the LiveObjects plugin. In order to use Live Objects, you must pass in the plugin via client options. +#### Using the plugin -```javascript +LiveObjects functionality is supported for Realtime clients via the LiveObjects plugin. In order to use LiveObjects, you must pass in the plugin via client options. + +```typescript import * as Ably from 'ably'; import LiveObjects from 'ably/liveobjects'; @@ -610,7 +612,7 @@ Alternatively, you can load the LiveObjects plugin directly in your HTML using ` When loaded this way, the LiveObjects plugin will be available on the global object via the `AblyLiveObjectsPlugin` property, so you will need to pass it to the Ably instance as follows: -```javascript +```typescript const client = new Ably.Realtime({ ...options, plugins: { LiveObjects: AblyLiveObjectsPlugin }, @@ -619,7 +621,285 @@ const client = new Ably.Realtime({ The LiveObjects plugin is developed as part of the Ably client library, so it is available for the same versions as the Ably client library itself. It also means that it follows the same semantic versioning rules as they were defined for [the Ably client library](#for-browsers). For example, to lock into a major or minor version of the LiveObjects plugin, you can specify a specific version number such as https://cdn.ably.com/lib/liveobjects.umd.min-2.js for all v2._ versions, or https://cdn.ably.com/lib/liveobjects.umd.min-2.4.js for all v2.4._ versions, or you can lock into a single release with https://cdn.ably.com/lib/liveobjects.umd.min-2.4.0.js. Note you can load the non-minified version by omitting `.min` from the URL such as https://cdn.ably.com/lib/liveobjects.umd-2.js. -For more information about Live Objects product, see the [Ably Live Objects documentation](https://ably.com/docs/products/liveobjects). +For more information about the LiveObjects product, see the [Ably LiveObjects documentation](https://ably.com/docs/products/liveobjects). + +#### State Channel Modes + +To use the LiveObjects feature, clients must attach to a channel with the correct channel mode: + +- `state_subscribe` - required to retrieve state of Live Objects for a channel +- `state_publish` - required to create new and modify existing Live Objects on a channel + +```typescript +const client = new Ably.Realtime({ + // authentication options + ...options, + plugins: { LiveObjects }, +}); +const channelOptions = { modes: ['state_subscribe', 'state_publish'] }; +const channel = client.channels.get('my_live_objects_channel', channelOptions); +const liveObjects = channel.liveObjects; +``` + +The authentication token must include corresponding capabilities for the client to interact with LiveObjects. + +#### Getting the Root Object + +The root object represents the top-level entry point for LiveObjects state within a channel. It gives access to all other nested Live Objects. + +```typescript +const root = await liveObjects.getRoot(); +``` + +The root object is a `LiveMap` instance and serves as the starting point for storing and organizing LiveObjects state. + +#### Live Object Types + +LiveObjects currently supports two primary data structures; `LiveMap` and `LiveCounter`. + +`LiveMap` - A key/value map data structure, similar to a JavaScript `Map`, where all changes are synchronized across clients in realtime. It allows you to store primitive values and other Live Objects, enabling a composable state. + +You can use `LiveMap` as follows: + +```typescript +// root object is a LiveMap +const root = await liveObjects.getRoot(); + +// you can read values for a key with .get +root.get('foo'); +root.get('bar'); + +// get a number of key/value pairs in a map with .size +root.size(); + +// set keys on a map with .set +// different data types are supported +await root.set('foo', 'Alice'); +await root.set('bar', 1); +await root.set('baz', true); +await root.set('qux', new Uint8Array([21, 31])); +// as well as other live objects +const counter = await liveObjects.createCounter(); +await root.set('quux', counter); + +// and you can remove keys with .remove +await root.remove('name'); +``` + +`LiveCounter` - A counter that can be incremented or decremented and is synchronized across clients in realtime + +You can use `LiveCounter` as follows: + +```typescript +const counter = await liveObjects.createCounter(); + +// you can get current value of a counter with .value +counter.value(); + +// and change its value with .increment or .decrement +await counter.increment(5); +await counter.decrement(2); +``` + +#### Subscribing to Updates + +Subscribing to updates on Live Objects allows you to receive changes made by other clients in realtime. Since multiple clients may modify the same Live Objects state, subscribing ensures that your application reacts to external updates as soon as they are received. + +Additionally, mutation methods such as `LiveMap.set`, `LiveCounter.increment`, and `LiveCounter.decrement` do not directly edit the current state of the object locally. Instead, they send the intended operation to the Ably system, and the change is applied to the local object only when the corresponding realtime operation is echoed back to the client. This means that the state you retrieve immediately after a mutation may not reflect the latest updates yet. + +You can subscribe to updates on all Live Objects using subscription listeners as follows: + +```typescript +const root = await liveObjects.getRoot(); + +// subscribe to updates on a LiveMap +root.subscribe((update: LiveMapUpdate) => { + console.log('LiveMap "name" key:', root.get('name')); // can read the current value for a key in a map inside this callback + console.log('LiveMap update details:', update); // and can get update details from the provided update object +}); + +// subscribe to updates on a LiveCounter +const counter = await liveObjects.createCounter(); +counter.subscribe((update: LiveCounterUpdate) => { + console.log('LiveCounter new value:', counter.value()); // can read the current value of the counter inside this callback + console.log('LiveCounter update details:', update); // and can get update details from the provided update object +}); + +// perform operations on LiveMap and LiveCounter +await root.set('name', 'Alice'); +// LiveMap "name" key: Alice +// LiveMap update details: { update: { name: 'updated' } } + +await root.remove('name'); +// LiveMap "name" key: undefined +// LiveMap update details: { update: { name: 'removed' } } + +await counter.increment(5); +// LiveCounter new value: 5 +// LiveCounter update details: { update: { inc: 5 } } + +await counter.decrement(2); +// LiveCounter new value: 3 +// LiveCounter update details: { update: { inc: -2 } } +``` + +You can deregister subscription listeners as follows: + +```typescript +// use dedicated unsubscribe function from the .subscribe call +const { unsubscribe } = root.subscribe(() => {}); +unsubscribe(); + +// call .unsubscribe with a listener reference +const listener = () => {}; +root.subscribe(listener); +root.unsubscribe(listener); + +// deregister all listeners using .unsubscribeAll +root.unsubscribeAll(); +``` + +#### Creating New Objects + +New `LiveMap` and `LiveCounter` instances can be created as follows: + +```typescript +const counter = await liveObjects.createCounter(123); // with optional initial counter value +const map = await liveObjects.createMap({ key: 'value' }); // with optional initial map entries +``` + +To persist them within the LiveObjects state, they must be assigned to a parent `LiveMap` that is connected to the root object through the object hierarchy: + +```typescript +const root = await liveObjects.getRoot(); + +const counter = await liveObjects.createCounter(); +const map = await liveObjects.createMap({ counter }); +const outerMap = await liveObjects.createMap({ map }); + +await root.set('outerMap', outerMap); + +// resulting structure: +// root (LiveMap) +// └── outerMap (LiveMap) +// └── map (LiveMap) +// └── counter (LiveCounter) +``` + +#### Batch Operations + +Batching allows multiple operations to be grouped into a single message that is sent to the Ably service. This allows batched operations to be applied atomically together. + +Within a batch callback, the `BatchContext` instance provides wrapper objects around regular Live Objects with a synchronous API for storing changes in the batch context. + +```typescript +await liveObjects.batch((ctx) => { + const root = ctx.getRoot(); + + root.set('foo', 'bar'); + root.set('baz', 42); + + const counter = root.get('counter'); + counter.increment(5); + + // batched operations are sent to the Ably service when the batch callback returns +}); +``` + +#### Lifecycle Events + +Live Objects emit lifecycle events to signal critical state changes, such as synchronization progress and object deletions. + +**Synchronization Events** - the `syncing` and `synced` events notify when the LiveObjects state is being synchronized with the Ably service. These events can be useful for displaying loading indicators, preventing user edits during synchronization, or triggering application logic when the data was loaded for the first time. + +```typescript +liveObjects.on('syncing', () => { + console.log('LiveObjects state is syncing...'); + // Example: Show a loading indicator +}); + +liveObjects.on('synced', () => { + console.log('LiveObjects state has been synced.'); + // Example: Hide loading indicator +}); +``` + +**Object Deletion Events** - objects that have been orphaned for a long period (i.e., not connected to the state tree graph by being set as a key in a map accessible from the root map object) will eventually be deleted. Once a Live Object is deleted, it can no longer be interacted with. You should avoid accessing its data or trying to update its value and you should remove all references to the deleted object in your application. + +```typescript +const root = await liveObjects.getRoot(); +const counter = root.get('counter'); + +counter.on('deleted', () => { + console.log('Live Object has been deleted.'); + // Example: Remove references to the object from the application +}); +``` + +To unsubscribe from lifecycle events: + +```typescript +// same API for channel.liveObjects and LiveObject instances +// use dedicated off function from the .on call +const { off } = liveObjects.on('synced', () => {}); +off(); + +// call .off with an event name and a listener reference +const listener = () => {}; +liveObjects.on('synced', listener); +liveObjects.off('synced', listener); + +// deregister all listeners using .offAll +liveObjects.offAll(); +``` + +#### Typing LiveObjects + +You can provide your own TypeScript typings for LiveObjects by providing a globally defined `LiveObjectsTypes` interface. + +```typescript +// file: ably.config.d.ts +import { LiveCounter, LiveMap } from 'ably'; + +type MyCustomRoot = { + map: LiveMap<{ + foo: string; + counter: LiveCounter; + }>; +}; + +declare global { + export interface LiveObjectsTypes { + root: MyCustomRoot; + } +} +``` + +This will enable code completion and editor hints when interacting with the LiveObjects API: + +```typescript +const root = await liveObjects.getRoot(); // uses types defined by global LiveObjectsTypes interface by default + +const map = root.get('map'); // LiveMap<{ foo: string; counter: LiveCounter }> +map.set('foo', 1); // TypeError +map.get('counter').value(); // autocompletion for counter method names +``` + +You can also provide typings for the LiveObjects state tree when calling the `liveObjects.getRoot` method, allowing you to have different state typings for different channels: + +```typescript +type ReactionsRoot = { + hearts: LiveCounter; + likes: LiveCounter; +}; + +type PollsRoot = { + currentPoll: LiveMap; +}; + +const reactionsRoot = await reactionsChannel.liveObjects.getRoot(); +const pollsRoot = await pollsChannel.liveObjects.getRoot(); +``` ## Delta Plugin diff --git a/ably.d.ts b/ably.d.ts index 4ed3c0811c..99746e6621 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -2056,11 +2056,11 @@ export declare interface PushChannel { */ declare namespace LiveObjectsEvents { /** - * The local Live Objects state is currently being synchronized with the Ably service. + * The local LiveObjects state is currently being synchronized with the Ably service. */ type SYNCING = 'syncing'; /** - * The local Live Objects state has been synchronized with the Ably service. + * The local LiveObjects state has been synchronized with the Ably service. */ type SYNCED = 'synced'; } @@ -2075,7 +2075,7 @@ export type LiveObjectsEvent = LiveObjectsEvents.SYNCED | LiveObjectsEvents.SYNC */ declare namespace LiveObjectLifecycleEvents { /** - * Indicates that the object has been deleted from the Live Objects pool and should no longer be interacted with. + * Indicates that the object has been deleted from the LiveObjects pool and should no longer be interacted with. */ type DELETED = 'deleted'; } From 007477de73a7d2d69b5629dd929416df335a5cbe Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 25 Feb 2025 09:13:32 +0000 Subject: [PATCH 131/166] Refactor channel modes tests --- test/realtime/live_objects.test.js | 109 +++++++++++++---------------- 1 file changed, 50 insertions(+), 59 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 52344122d6..0e79fe8001 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -4146,71 +4146,55 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], } }); - const expectToThrowMissingStateMode = async ({ liveObjects, map, counter }) => { - await expectToThrowAsync( - async () => liveObjects.getRoot(), - '"state_subscribe" channel mode must be set for this operation', - ); - await expectToThrowAsync( - async () => liveObjects.batch(), - '"state_publish" channel mode must be set for this operation', - ); - await expectToThrowAsync( - async () => liveObjects.createMap(), - '"state_publish" channel mode must be set for this operation', - ); - await expectToThrowAsync( - async () => liveObjects.createCounter(), - '"state_publish" channel mode must be set for this operation', - ); + const expectAccessApiToThrow = async ({ liveObjects, map, counter, errorMsg }) => { + await expectToThrowAsync(async () => liveObjects.getRoot(), errorMsg); - expect(() => counter.value()).to.throw('"state_subscribe" channel mode must be set for this operation'); - await expectToThrowAsync( - async () => counter.increment(), - '"state_publish" channel mode must be set for this operation', - ); - await expectToThrowAsync( - async () => counter.decrement(), - '"state_publish" channel mode must be set for this operation', - ); + expect(() => counter.value()).to.throw(errorMsg); - expect(() => map.get()).to.throw('"state_subscribe" channel mode must be set for this operation'); - expect(() => map.size()).to.throw('"state_subscribe" channel mode must be set for this operation'); - await expectToThrowAsync(async () => map.set(), '"state_publish" channel mode must be set for this operation'); - await expectToThrowAsync( - async () => map.remove(), - '"state_publish" channel mode must be set for this operation', - ); + expect(() => map.get()).to.throw(errorMsg); + expect(() => map.size()).to.throw(errorMsg); for (const obj of [map, counter]) { - expect(() => obj.subscribe()).to.throw('"state_subscribe" channel mode must be set for this operation'); - expect(() => obj.unsubscribe(() => {})).not.to.throw( - '"state_subscribe" channel mode must be set for this operation', - ); // note: this should not throw - expect(() => obj.unsubscribe(() => {})).not.to.throw( - '"state_publish" channel mode must be set for this operation', - ); // note: this should not throw - expect(() => obj.unsubscribeAll()).not.to.throw( - '"state_subscribe" channel mode must be set for this operation', - ); // note: this should not throw - expect(() => obj.unsubscribeAll()).not.to.throw( - '"state_publish" channel mode must be set for this operation', - ); // note: this should not throw + expect(() => obj.subscribe()).to.throw(errorMsg); + expect(() => obj.unsubscribe(() => {})).not.to.throw(); // this should not throw + expect(() => obj.unsubscribeAll()).not.to.throw(); // this should not throw + } + }; + + const expectWriteApiToThrow = async ({ liveObjects, map, counter, errorMsg }) => { + await expectToThrowAsync(async () => liveObjects.batch(), errorMsg); + await expectToThrowAsync(async () => liveObjects.createMap(), errorMsg); + await expectToThrowAsync(async () => liveObjects.createCounter(), errorMsg); + + await expectToThrowAsync(async () => counter.increment(), errorMsg); + await expectToThrowAsync(async () => counter.decrement(), errorMsg); + + await expectToThrowAsync(async () => map.set(), errorMsg); + await expectToThrowAsync(async () => map.remove(), errorMsg); + + for (const obj of [map, counter]) { + expect(() => obj.unsubscribe(() => {})).not.to.throw(); // this should not throw + expect(() => obj.unsubscribeAll()).not.to.throw(); // this should not throw } }; /** Make sure to call this inside the batch method as batch objects can't be interacted with outside the batch callback */ - const expectToThrowMissingStateModeInBatchContext = ({ ctx, map, counter }) => { - expect(() => ctx.getRoot()).to.throw('"state_subscribe" channel mode must be set for this operation'); + const expectAccessBatchApiToThrow = ({ ctx, map, counter, errorMsg }) => { + expect(() => ctx.getRoot()).to.throw(errorMsg); + + expect(() => counter.value()).to.throw(errorMsg); + + expect(() => map.get()).to.throw(errorMsg); + expect(() => map.size()).to.throw(errorMsg); + }; - expect(() => counter.value()).to.throw('"state_subscribe" channel mode must be set for this operation'); - expect(() => counter.increment()).to.throw('"state_publish" channel mode must be set for this operation'); - expect(() => counter.decrement()).to.throw('"state_publish" channel mode must be set for this operation'); + /** Make sure to call this inside the batch method as batch objects can't be interacted with outside the batch callback */ + const expectWriteBatchApiToThrow = ({ ctx, map, counter, errorMsg }) => { + expect(() => counter.increment()).to.throw(errorMsg); + expect(() => counter.decrement()).to.throw(errorMsg); - expect(() => map.get()).to.throw('"state_subscribe" channel mode must be set for this operation'); - expect(() => map.size()).to.throw('"state_subscribe" channel mode must be set for this operation'); - expect(() => map.set()).to.throw('"state_publish" channel mode must be set for this operation'); - expect(() => map.remove()).to.throw('"state_publish" channel mode must be set for this operation'); + expect(() => map.set()).to.throw(errorMsg); + expect(() => map.remove()).to.throw(errorMsg); }; const missingChannelModesScenarios = [ @@ -4225,9 +4209,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const counter = ctx.getRoot().get('counter'); // now simulate missing modes channel.modes = []; - expectToThrowMissingStateModeInBatchContext({ ctx, map, counter }); + + expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: '"state_subscribe" channel mode' }); + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: '"state_publish" channel mode' }); }); - await expectToThrowMissingStateMode({ liveObjects, map, counter }); + + await expectAccessApiToThrow({ liveObjects, map, counter, errorMsg: '"state_subscribe" channel mode' }); + await expectWriteApiToThrow({ liveObjects, map, counter, errorMsg: '"state_publish" channel mode' }); }, }, @@ -4246,9 +4234,12 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], helper.recordPrivateApi('write.channel.channelOptions.modes'); channel.channelOptions.modes = []; - expectToThrowMissingStateModeInBatchContext({ ctx, map, counter }); + expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: '"state_subscribe" channel mode' }); + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: '"state_publish" channel mode' }); }); - await expectToThrowMissingStateMode({ liveObjects, map, counter }); + + await expectAccessApiToThrow({ liveObjects, map, counter, errorMsg: '"state_subscribe" channel mode' }); + await expectWriteApiToThrow({ liveObjects, map, counter, errorMsg: '"state_publish" channel mode' }); }, }, ]; From 717ff1679b25b9593c22cf3fa05b38fa8d736f26 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 25 Feb 2025 09:09:16 +0000 Subject: [PATCH 132/166] Implement LiveObjects behavior under different channel states Resolves PUB-1030 --- src/plugins/liveobjects/batchcontext.ts | 2 +- .../liveobjects/batchcontextlivecounter.ts | 6 +- .../liveobjects/batchcontextlivemap.ts | 8 +- src/plugins/liveobjects/livecounter.ts | 6 +- src/plugins/liveobjects/livemap.ts | 8 +- src/plugins/liveobjects/liveobject.ts | 12 +-- src/plugins/liveobjects/liveobjects.ts | 33 ++++--- test/realtime/live_objects.test.js | 86 ++++++++++++++++++- 8 files changed, 123 insertions(+), 38 deletions(-) diff --git a/src/plugins/liveobjects/batchcontext.ts b/src/plugins/liveobjects/batchcontext.ts index e96378248a..1d2f6aef72 100644 --- a/src/plugins/liveobjects/batchcontext.ts +++ b/src/plugins/liveobjects/batchcontext.ts @@ -24,7 +24,7 @@ export class BatchContext { } getRoot(): BatchContextLiveMap { - this._liveObjects.throwIfMissingStateSubscribeMode(); + this._liveObjects.throwIfInvalidAccessApiConfiguration(); this.throwIfClosed(); return this.getWrappedObject(ROOT_OBJECT_ID) as BatchContextLiveMap; } diff --git a/src/plugins/liveobjects/batchcontextlivecounter.ts b/src/plugins/liveobjects/batchcontextlivecounter.ts index 3528c3ecbd..c2a673f958 100644 --- a/src/plugins/liveobjects/batchcontextlivecounter.ts +++ b/src/plugins/liveobjects/batchcontextlivecounter.ts @@ -15,20 +15,20 @@ export class BatchContextLiveCounter { } value(): number { - this._liveObjects.throwIfMissingStateSubscribeMode(); + this._liveObjects.throwIfInvalidAccessApiConfiguration(); this._batchContext.throwIfClosed(); return this._counter.value(); } increment(amount: number): void { - this._liveObjects.throwIfMissingStatePublishMode(); + this._liveObjects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); const stateMessage = LiveCounter.createCounterIncMessage(this._liveObjects, this._counter.getObjectId(), amount); this._batchContext.queueStateMessage(stateMessage); } decrement(amount: number): void { - this._liveObjects.throwIfMissingStatePublishMode(); + this._liveObjects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); // do an explicit type safety check here before negating the amount value, // so we don't unintentionally change the type sent by a user diff --git a/src/plugins/liveobjects/batchcontextlivemap.ts b/src/plugins/liveobjects/batchcontextlivemap.ts index 9d6fe782dc..1173cb45ef 100644 --- a/src/plugins/liveobjects/batchcontextlivemap.ts +++ b/src/plugins/liveobjects/batchcontextlivemap.ts @@ -12,7 +12,7 @@ export class BatchContextLiveMap { ) {} get(key: TKey): T[TKey] | undefined { - this._liveObjects.throwIfMissingStateSubscribeMode(); + this._liveObjects.throwIfInvalidAccessApiConfiguration(); this._batchContext.throwIfClosed(); const value = this._map.get(key); if (value instanceof LiveObject) { @@ -23,20 +23,20 @@ export class BatchContextLiveMap { } size(): number { - this._liveObjects.throwIfMissingStateSubscribeMode(); + this._liveObjects.throwIfInvalidAccessApiConfiguration(); this._batchContext.throwIfClosed(); return this._map.size(); } set(key: TKey, value: T[TKey]): void { - this._liveObjects.throwIfMissingStatePublishMode(); + this._liveObjects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); const stateMessage = LiveMap.createMapSetMessage(this._liveObjects, this._map.getObjectId(), key, value); this._batchContext.queueStateMessage(stateMessage); } remove(key: TKey): void { - this._liveObjects.throwIfMissingStatePublishMode(); + this._liveObjects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); const stateMessage = LiveMap.createMapRemoveMessage(this._liveObjects, this._map.getObjectId(), key); this._batchContext.queueStateMessage(stateMessage); diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 8730f57b24..5762a8cdbb 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -123,7 +123,7 @@ export class LiveCounter extends LiveObject } value(): number { - this._liveObjects.throwIfMissingStateSubscribeMode(); + this._liveObjects.throwIfInvalidAccessApiConfiguration(); return this._dataRef.data; } @@ -137,7 +137,7 @@ export class LiveCounter extends LiveObject * @returns A promise which resolves upon receiving the ACK message for the published operation message. */ async increment(amount: number): Promise { - this._liveObjects.throwIfMissingStatePublishMode(); + this._liveObjects.throwIfInvalidWriteApiConfiguration(); const stateMessage = LiveCounter.createCounterIncMessage(this._liveObjects, this.getObjectId(), amount); return this._liveObjects.publish([stateMessage]); } @@ -146,7 +146,7 @@ export class LiveCounter extends LiveObject * An alias for calling {@link LiveCounter.increment | LiveCounter.increment(-amount)} */ async decrement(amount: number): Promise { - this._liveObjects.throwIfMissingStatePublishMode(); + this._liveObjects.throwIfInvalidWriteApiConfiguration(); // do an explicit type safety check here before negating the amount value, // so we don't unintentionally change the type sent by a user if (typeof amount !== 'number' || !isFinite(amount)) { diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 637c5eeed0..6dd3410ff3 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -266,7 +266,7 @@ export class LiveMap extends LiveObject(key: TKey): T[TKey] | undefined { - this._liveObjects.throwIfMissingStateSubscribeMode(); + this._liveObjects.throwIfInvalidAccessApiConfiguration(); if (this.isTombstoned()) { return undefined as T[TKey]; @@ -305,7 +305,7 @@ export class LiveMap extends LiveObject extends LiveObject(key: TKey, value: T[TKey]): Promise { - this._liveObjects.throwIfMissingStatePublishMode(); + this._liveObjects.throwIfInvalidWriteApiConfiguration(); const stateMessage = LiveMap.createMapSetMessage(this._liveObjects, this.getObjectId(), key, value); return this._liveObjects.publish([stateMessage]); } @@ -356,7 +356,7 @@ export class LiveMap extends LiveObject(key: TKey): Promise { - this._liveObjects.throwIfMissingStatePublishMode(); + this._liveObjects.throwIfInvalidWriteApiConfiguration(); const stateMessage = LiveMap.createMapRemoveMessage(this._liveObjects, this.getObjectId(), key); return this._liveObjects.publish([stateMessage]); } diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/liveobjects/liveobject.ts index 88a4d9953b..4485b5be86 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/liveobjects/liveobject.ts @@ -77,7 +77,7 @@ export abstract class LiveObject< } subscribe(listener: (update: TUpdate) => void): SubscribeResponse { - this._liveObjects.throwIfMissingStateSubscribeMode(); + this._liveObjects.throwIfInvalidAccessApiConfiguration(); this._subscriptions.on(LiveObjectSubscriptionEvent.updated, listener); @@ -89,7 +89,7 @@ export abstract class LiveObject< } unsubscribe(listener: (update: TUpdate) => void): void { - // can allow calling this public method without checking for state modes on the channel as the result of this method is not dependant on them + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. // current implementation of the EventEmitter will remove all listeners if .off is called without arguments or with nullish arguments. // or when called with just an event argument, it will remove all listeners for the event. @@ -102,12 +102,12 @@ export abstract class LiveObject< } unsubscribeAll(): void { - // can allow calling this public method without checking for state modes on the channel as the result of this method is not dependant on them + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. this._subscriptions.off(LiveObjectSubscriptionEvent.updated); } on(event: LiveObjectLifecycleEvent, callback: LiveObjectLifecycleEventCallback): OnLiveObjectLifecycleEventResponse { - // we don't require any specific channel mode to be set to call this public method + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. this._lifecycleEvents.on(event, callback); const off = () => { @@ -118,7 +118,7 @@ export abstract class LiveObject< } off(event: LiveObjectLifecycleEvent, callback: LiveObjectLifecycleEventCallback): void { - // we don't require any specific channel mode to be set to call this public method + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. // prevent accidentally calling .off without any arguments on an EventEmitter and removing all callbacks if (this._client.Utils.isNil(event) && this._client.Utils.isNil(callback)) { @@ -129,7 +129,7 @@ export abstract class LiveObject< } offAll(): void { - // we don't require any specific channel mode to be set to call this public method + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. this._lifecycleEvents.off(); } diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index c8dbc60b74..1ceed73aba 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -71,7 +71,7 @@ export class LiveObjects { * This is useful when working with LiveObjects on multiple channels with different underlying data. */ async getRoot(): Promise> { - this.throwIfMissingStateSubscribeMode(); + this.throwIfInvalidAccessApiConfiguration(); // if we're not synced yet, wait for SYNC sequence to finish before returning root if (this._state !== LiveObjectsState.synced) { @@ -85,7 +85,7 @@ export class LiveObjects { * Provides access to the synchronous write API for LiveObjects that can be used to batch multiple operations together in a single channel message. */ async batch(callback: BatchCallback): Promise { - this.throwIfMissingStatePublishMode(); + this.throwIfInvalidWriteApiConfiguration(); const root = await this.getRoot(); const context = new BatchContext(this, root); @@ -107,7 +107,7 @@ export class LiveObjects { * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with an object containing provided data. */ async createMap(entries?: T): Promise> { - this.throwIfMissingStatePublishMode(); + this.throwIfInvalidWriteApiConfiguration(); const stateMessage = await LiveMap.createMapCreateMessage(this, entries); const objectId = stateMessage.operation?.objectId!; @@ -139,7 +139,7 @@ export class LiveObjects { * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with an object containing provided data. */ async createCounter(count?: number): Promise { - this.throwIfMissingStatePublishMode(); + this.throwIfInvalidWriteApiConfiguration(); const stateMessage = await LiveCounter.createCounterCreateMessage(this, count); const objectId = stateMessage.operation?.objectId!; @@ -162,7 +162,7 @@ export class LiveObjects { } on(event: LiveObjectsEvent, callback: LiveObjectsEventCallback): OnLiveObjectsEventResponse { - // we don't require any specific channel mode to be set to call this public method + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. this._eventEmitterPublic.on(event, callback); const off = () => { @@ -173,7 +173,7 @@ export class LiveObjects { } off(event: LiveObjectsEvent, callback: LiveObjectsEventCallback): void { - // we don't require any specific channel mode to be set to call this public method + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. // prevent accidentally calling .off without any arguments on an EventEmitter and removing all callbacks if (this._client.Utils.isNil(event) && this._client.Utils.isNil(callback)) { @@ -184,7 +184,7 @@ export class LiveObjects { } offAll(): void { - // we don't require any specific channel mode to be set to call this public method + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. this._eventEmitterPublic.off(); } @@ -284,11 +284,8 @@ export class LiveObjects { case 'detached': case 'failed': - // TODO: do something - break; - - case 'suspended': - // TODO: do something + this._liveObjectsPool.reset(); + this._syncLiveObjectsDataPool.reset(); break; } } @@ -322,15 +319,17 @@ export class LiveObjects { /** * @internal */ - throwIfMissingStateSubscribeMode(): void { + throwIfInvalidAccessApiConfiguration(): void { this._throwIfMissingChannelMode('state_subscribe'); + this._throwIfInChannelState(['detached', 'failed']); } /** * @internal */ - throwIfMissingStatePublishMode(): void { + throwIfInvalidWriteApiConfiguration(): void { this._throwIfMissingChannelMode('state_publish'); + this._throwIfInChannelState(['detached', 'failed', 'suspended']); } private _startNewSync(syncId?: string, syncCursor?: string): void { @@ -493,4 +492,10 @@ export class LiveObjects { this._eventEmitterPublic.emit(event); } } + + private _throwIfInChannelState(channelState: API.ChannelState[]): void { + if (channelState.includes(this._channel.state)) { + throw this._client.ErrorInfo.fromValues(this._channel.invalidStateError()); + } + } } diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 0e79fe8001..8cb7ffab64 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -4004,6 +4004,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ); await counterCreatedPromise; + helper.recordPrivateApi('call.LiveObjects._liveObjectsPool.get'); expect(liveObjects._liveObjectsPool.get(objectId), 'Check object exists in the pool after creation').to .exist; @@ -4197,7 +4198,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(() => map.remove()).to.throw(errorMsg); }; - const missingChannelModesScenarios = [ + const channelConfigurationScenarios = [ { description: 'public API throws missing state modes error when attached without correct state modes', action: async (ctx) => { @@ -4242,16 +4243,95 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await expectWriteApiToThrow({ liveObjects, map, counter, errorMsg: '"state_publish" channel mode' }); }, }, + + { + description: 'public API throws invalid channel state error when channel DETACHED', + action: async (ctx) => { + const { liveObjects, channel, map, counter, helper } = ctx; + + // obtain batch context with valid channel state first + await liveObjects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate channel state change + helper.recordPrivateApi('call.channel.requestState'); + channel.requestState('detached'); + + expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: 'failed as channel state is detached' }); + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: 'failed as channel state is detached' }); + }); + + await expectAccessApiToThrow({ + liveObjects, + map, + counter, + errorMsg: 'failed as channel state is detached', + }); + await expectWriteApiToThrow({ liveObjects, map, counter, errorMsg: 'failed as channel state is detached' }); + }, + }, + + { + description: 'public API throws invalid channel state error when channel FAILED', + action: async (ctx) => { + const { liveObjects, channel, map, counter, helper } = ctx; + + // obtain batch context with valid channel state first + await liveObjects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate channel state change + helper.recordPrivateApi('call.channel.requestState'); + channel.requestState('failed'); + + expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: 'failed as channel state is failed' }); + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: 'failed as channel state is failed' }); + }); + + await expectAccessApiToThrow({ + liveObjects, + map, + counter, + errorMsg: 'failed as channel state is failed', + }); + await expectWriteApiToThrow({ liveObjects, map, counter, errorMsg: 'failed as channel state is failed' }); + }, + }, + + { + description: 'public write API throws invalid channel state error when channel SUSPENDED', + action: async (ctx) => { + const { liveObjects, channel, map, counter, helper } = ctx; + + // obtain batch context with valid channel state first + await liveObjects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate channel state change + helper.recordPrivateApi('call.channel.requestState'); + channel.requestState('suspended'); + + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: 'failed as channel state is suspended' }); + }); + + await expectWriteApiToThrow({ + liveObjects, + map, + counter, + errorMsg: 'failed as channel state is suspended', + }); + }, + }, ]; /** @nospec */ - forScenarios(this, missingChannelModesScenarios, async function (helper, scenario, clientOptions, channelName) { + forScenarios(this, channelConfigurationScenarios, async function (helper, scenario, clientOptions, channelName) { const liveObjectsHelper = new LiveObjectsHelper(helper); const client = RealtimeWithLiveObjects(helper, clientOptions); await helper.monitorConnectionThenCloseAndFinish(async () => { // attach with correct channel modes so we can create liveobjects on the root for testing. - // each scenario will modify the underlying modes array to test specific behavior + // some scenarios will modify the underlying modes array to test specific behavior const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); const liveObjects = channel.liveObjects; From dff82cd5d6a55cfaff5d9d2afef33c39d922819a Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 4 Mar 2025 07:49:37 +0000 Subject: [PATCH 133/166] Refactor live objects tests to reuse existing methods --- test/realtime/live_objects.test.js | 40 ++++++++++++++++++------------ 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index 8cb7ffab64..b9a7a55067 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -3373,14 +3373,18 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], savedCtxMap = ctxRoot.get('map'); }); - expect(() => savedCtx.getRoot()).to.throw('Batch is closed'); - expect(() => savedCtxCounter.value()).to.throw('Batch is closed'); - expect(() => savedCtxCounter.increment()).to.throw('Batch is closed'); - expect(() => savedCtxCounter.decrement()).to.throw('Batch is closed'); - expect(() => savedCtxMap.get()).to.throw('Batch is closed'); - expect(() => savedCtxMap.size()).to.throw('Batch is closed'); - expect(() => savedCtxMap.set()).to.throw('Batch is closed'); - expect(() => savedCtxMap.remove()).to.throw('Batch is closed'); + expectAccessBatchApiToThrow({ + ctx: savedCtx, + map: savedCtxMap, + counter: savedCtxCounter, + errorMsg: 'Batch is closed', + }); + expectWriteBatchApiToThrow({ + ctx: savedCtx, + map: savedCtxMap, + counter: savedCtxCounter, + errorMsg: 'Batch is closed', + }); }, }, @@ -3418,14 +3422,18 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], } expect(caughtError, 'Check batch call failed with an error').to.exist; - expect(() => savedCtx.getRoot()).to.throw('Batch is closed'); - expect(() => savedCtxCounter.value()).to.throw('Batch is closed'); - expect(() => savedCtxCounter.increment()).to.throw('Batch is closed'); - expect(() => savedCtxCounter.decrement()).to.throw('Batch is closed'); - expect(() => savedCtxMap.get()).to.throw('Batch is closed'); - expect(() => savedCtxMap.size()).to.throw('Batch is closed'); - expect(() => savedCtxMap.set()).to.throw('Batch is closed'); - expect(() => savedCtxMap.remove()).to.throw('Batch is closed'); + expectAccessBatchApiToThrow({ + ctx: savedCtx, + map: savedCtxMap, + counter: savedCtxCounter, + errorMsg: 'Batch is closed', + }); + expectWriteBatchApiToThrow({ + ctx: savedCtx, + map: savedCtxMap, + counter: savedCtxCounter, + errorMsg: 'Batch is closed', + }); }, }, ]; From 24aba842dd63eca24da9d40364b2f81457e07a2c Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 4 Mar 2025 07:52:31 +0000 Subject: [PATCH 134/166] Add enumeration API to LiveMap and BatchContextLiveMap Resolves PUB-1235 --- README.md | 11 ++ ably.d.ts | 15 ++ .../liveobjects/batchcontextlivemap.ts | 18 +++ src/plugins/liveobjects/livemap.ts | 100 ++++++++---- test/realtime/live_objects.test.js | 145 ++++++++++++++++++ 5 files changed, 259 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 4124d939de..131b3990df 100644 --- a/README.md +++ b/README.md @@ -672,6 +672,17 @@ root.get('bar'); // get a number of key/value pairs in a map with .size root.size(); +// iterate over keys/values in a map +for (const [key, value] of root.entries()) { + /**/ +} +for (const key of root.keys()) { + /**/ +} +for (const value of root.values()) { + /**/ +} + // set keys on a map with .set // different data types are supported await root.set('foo', 'Alice'); diff --git a/ably.d.ts b/ably.d.ts index 99746e6621..ca2119baf5 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -2314,6 +2314,21 @@ export declare interface LiveMap extends LiveObject(): IterableIterator<[TKey, T[TKey]]>; + + /** + * Returns an iterable of keys in the map. + */ + keys(): IterableIterator; + + /** + * Returns an iterable of values in the map. + */ + values(): IterableIterator; + /** * Sends an operation to the Ably system to set a key on this `LiveMap` object to a specified value. * diff --git a/src/plugins/liveobjects/batchcontextlivemap.ts b/src/plugins/liveobjects/batchcontextlivemap.ts index 1173cb45ef..2e88b260f8 100644 --- a/src/plugins/liveobjects/batchcontextlivemap.ts +++ b/src/plugins/liveobjects/batchcontextlivemap.ts @@ -28,6 +28,24 @@ export class BatchContextLiveMap { return this._map.size(); } + *entries(): IterableIterator<[TKey, T[TKey]]> { + this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._batchContext.throwIfClosed(); + yield* this._map.entries(); + } + + *keys(): IterableIterator { + this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._batchContext.throwIfClosed(); + yield* this._map.keys(); + } + + *values(): IterableIterator { + this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._batchContext.throwIfClosed(); + yield* this._map.values(); + } + set(key: TKey, value: T[TKey]): void { this._liveObjects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 6dd3410ff3..6b662fb0b2 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -283,52 +283,50 @@ export class LiveMap extends LiveObject(): IterableIterator<[TKey, T[TKey]]> { this._liveObjects.throwIfInvalidAccessApiConfiguration(); - let size = 0; - for (const value of this._dataRef.data.values()) { - if (value.tombstone === true) { - // should not count removed entries + for (const [key, entry] of this._dataRef.data.entries()) { + if (this._isMapEntryTombstoned(entry)) { + // do not return tombstoned entries continue; } // data always exists for non-tombstoned elements - const data = value.data!; - if ('objectId' in data) { - const refObject = this._liveObjects.getPool().get(data.objectId); - - if (refObject?.isTombstoned()) { - // should not count tombstoned objects - continue; - } - } + const value = this._getResolvedValueFromStateData(entry.data!) as T[TKey]; + yield [key as TKey, value]; + } + } - size++; + *keys(): IterableIterator { + for (const [key] of this.entries()) { + yield key; } + } - return size; + *values(): IterableIterator { + for (const [_, value] of this.entries()) { + yield value; + } } /** @@ -792,4 +790,46 @@ export class LiveMap extends LiveObject { + const { root, liveObjectsHelper, channel } = ctx; + + const counterId1 = liveObjectsHelper.fakeCounterObjectId(); + const counterId2 = liveObjectsHelper.fakeCounterObjectId(); + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately + state: [ + liveObjectsHelper.counterObject({ + objectId: counterId1, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: false, + initialCount: 0, + }), + liveObjectsHelper.counterObject({ + objectId: counterId2, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialCount: 0, + }), + liveObjectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + materialisedEntries: { + counter1: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId1 } }, + counter2: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId2 } }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'bar' } }, + baz: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'qux' }, tombstone: true }, + }, + }), + ], + }); + + const counter1 = await root.get('counter1'); + + // enumeration methods should not count tombstoned entries + expect(root.size()).to.equal(2, 'Check LiveMap.size() returns expected number of keys'); + expect([...root.entries()]).to.deep.equal( + [ + ['counter1', counter1], + ['foo', 'bar'], + ], + 'Check LiveMap.entries() returns expected entries', + ); + expect([...root.keys()]).to.deep.equal(['counter1', 'foo'], 'Check LiveMap.keys() returns expected keys'); + expect([...root.values()]).to.deep.equal( + [counter1, 'bar'], + 'Check LiveMap.values() returns expected values', + ); + }, + }, + { + description: `BatchContextLiveMap enumeration`, + action: async (ctx) => { + const { root, liveObjectsHelper, channel, liveObjects } = ctx; + + const counterId1 = liveObjectsHelper.fakeCounterObjectId(); + const counterId2 = liveObjectsHelper.fakeCounterObjectId(); + await liveObjectsHelper.processStateObjectMessageOnChannel({ + channel, + syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately + state: [ + liveObjectsHelper.counterObject({ + objectId: counterId1, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: false, + initialCount: 0, + }), + liveObjectsHelper.counterObject({ + objectId: counterId2, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialCount: 0, + }), + liveObjectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + materialisedEntries: { + counter1: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId1 } }, + counter2: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId2 } }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'bar' } }, + baz: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'qux' }, tombstone: true }, + }, + }), + ], + }); + + const counter1 = await root.get('counter1'); + + await liveObjects.batch(async (ctx) => { + const ctxRoot = ctx.getRoot(); + + // enumeration methods should not count tombstoned entries + expect(ctxRoot.size()).to.equal(2, 'Check BatchContextLiveMap.size() returns expected number of keys'); + expect([...ctxRoot.entries()]).to.deep.equal( + [ + ['counter1', counter1], + ['foo', 'bar'], + ], + 'Check BatchContextLiveMap.entries() returns expected entries', + ); + expect([...ctxRoot.keys()]).to.deep.equal( + ['counter1', 'foo'], + 'Check BatchContextLiveMap.keys() returns expected keys', + ); + expect([...ctxRoot.values()]).to.deep.equal( + [counter1, 'bar'], + 'Check BatchContextLiveMap.values() returns expected values', + ); + }); + }, + }, + ]; + /** @nospec */ forScenarios( this, @@ -3446,6 +3584,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ...applyOperationsScenarios, ...applyOperationsDuringSyncScenarios, ...writeApiScenarios, + ...liveMapEnumerationScenarios, ], async function (helper, scenario, clientOptions, channelName) { const liveObjectsHelper = new LiveObjectsHelper(helper); @@ -4162,6 +4301,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(() => map.get()).to.throw(errorMsg); expect(() => map.size()).to.throw(errorMsg); + expect(() => [...map.entries()]).to.throw(errorMsg); + expect(() => [...map.keys()]).to.throw(errorMsg); + expect(() => [...map.values()]).to.throw(errorMsg); for (const obj of [map, counter]) { expect(() => obj.subscribe()).to.throw(errorMsg); @@ -4195,6 +4337,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(() => map.get()).to.throw(errorMsg); expect(() => map.size()).to.throw(errorMsg); + expect(() => [...map.entries()]).to.throw(errorMsg); + expect(() => [...map.keys()]).to.throw(errorMsg); + expect(() => [...map.values()]).to.throw(errorMsg); }; /** Make sure to call this inside the batch method as batch objects can't be interacted with outside the batch callback */ From af219388835cd9f39adb9c558a5d94a81dfb0a4e Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 5 Mar 2025 03:51:29 +0000 Subject: [PATCH 135/166] Rename "LiveObjects API" to "Objects API" Isolate API naming from Product naming by using `Objects` in API and code instead of the `LiveObjects` name. Based on the DR [1]. This commit does not change the names of the files (some of them are still using `liveObjects` name) in order to ensure that the changes will not get shadowed by the file renaming and make the review process easier. File names will be change in the following commit. Overview of changes in this commit: - replaced "LiveObjects" naming with "Objects" where it's not referencing the product name. Also tried to minimize the usage of word "state" when talking about Objects feature. For example, replaced phrases like "state on a channel" with "Objects on a channel", "LiveObjects state tree" with "channel Objects", and "channel state" with "channel Objects" or "Objects on a channel". - renamed the interface for `channel.objects` to "Objects". However, "LiveObject", "LiveMap", and "LiveCounter" (this one kept for parity) remain unchanged to avoid conflicts with native language classes like "Map" and "Object". - standardized references in comments and documentation to describe the "Objects" feature on a channel rather than using "LiveObjects" name. In rare cases, the product name is still used to clarify the relationship between the "Objects" feature/plugin and the "LiveObjects" product. - changed the REST API endpoint for LiveObjects to `/channels/{name}/objects`. [1] https://ably.atlassian.net/wiki/spaces/LOB/pages/3819896841/LODR-033+Isolating+API+naming+from+Product+naming+Renaming+state+and+liveobjects+to+objects+in+APIs --- Gruntfile.js | 12 +- README.md | 112 +- ably.d.ts | 92 +- grunt/esbuild/build.js | 24 +- liveobjects.d.ts | 16 +- package.json | 10 +- scripts/cdn_deploy.js | 2 +- scripts/moduleReport.ts | 18 +- src/common/lib/client/baserealtime.ts | 6 +- src/common/lib/client/modularplugins.ts | 4 +- src/common/lib/client/realtimechannel.ts | 38 +- src/common/lib/transport/comettransport.ts | 2 +- src/common/lib/transport/connectionmanager.ts | 3 +- src/common/lib/transport/protocol.ts | 2 +- src/common/lib/transport/transport.ts | 2 +- .../lib/transport/websockettransport.ts | 2 +- src/common/lib/types/protocolmessage.ts | 49 +- src/plugins/index.d.ts | 4 +- src/plugins/liveobjects/batchcontext.ts | 20 +- .../liveobjects/batchcontextlivecounter.ts | 14 +- .../liveobjects/batchcontextlivemap.ts | 22 +- src/plugins/liveobjects/index.ts | 6 +- src/plugins/liveobjects/livecounter.ts | 32 +- src/plugins/liveobjects/livemap.ts | 65 +- src/plugins/liveobjects/liveobject.ts | 10 +- src/plugins/liveobjects/liveobjects.ts | 118 +- src/plugins/liveobjects/liveobjectspool.ts | 16 +- .../liveobjects/syncliveobjectsdatapool.ts | 14 +- test/common/globals/named_dependencies.js | 8 +- test/common/modules/live_objects_helper.js | 14 +- test/common/modules/private_api_recorder.js | 12 +- test/package/browser/template/README.md | 2 +- .../server/resources/index-liveobjects.html | 2 +- .../browser/template/src/ably.config.d.ts | 2 +- .../browser/template/src/index-liveobjects.ts | 24 +- .../browser/template/test/lib/package.test.ts | 2 +- test/realtime/live_objects.test.js | 1229 ++++++++--------- typedoc.json | 2 +- 38 files changed, 975 insertions(+), 1037 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index 747c74d627..7ba8985289 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -73,7 +73,7 @@ module.exports = function (grunt) { }); }); - grunt.registerTask('build', ['webpack:all', 'build:browser', 'build:node', 'build:push', 'build:liveobjects']); + grunt.registerTask('build', ['webpack:all', 'build:browser', 'build:node', 'build:push', 'build:objects']); grunt.registerTask('all', ['build', 'requirejs']); @@ -138,13 +138,13 @@ module.exports = function (grunt) { }); }); - grunt.registerTask('build:liveobjects', function () { + grunt.registerTask('build:objects', function () { var done = this.async(); Promise.all([ - esbuild.build(esbuildConfig.liveObjectsPluginConfig), - esbuild.build(esbuildConfig.liveObjectsPluginCdnConfig), - esbuild.build(esbuildConfig.minifiedLiveObjectsPluginCdnConfig), + esbuild.build(esbuildConfig.objectsPluginConfig), + esbuild.build(esbuildConfig.objectsPluginCdnConfig), + esbuild.build(esbuildConfig.minifiedObjectsPluginCdnConfig), ]) .then(() => { done(true); @@ -157,7 +157,7 @@ module.exports = function (grunt) { grunt.registerTask('test:webserver', 'Launch the Mocha test web server on http://localhost:3000/', [ 'build:browser', 'build:push', - 'build:liveobjects', + 'build:objects', 'checkGitSubmodules', 'mocha:webserver', ]); diff --git a/README.md b/README.md index 131b3990df..562be82385 100644 --- a/README.md +++ b/README.md @@ -588,82 +588,82 @@ For more information on publishing push notifications over Ably, see the [Ably p ### LiveObjects -#### Using the plugin +#### Using the Objects plugin -LiveObjects functionality is supported for Realtime clients via the LiveObjects plugin. In order to use LiveObjects, you must pass in the plugin via client options. +LiveObjects functionality is supported for Realtime clients via the Objects plugin. In order to use Objects on a channel, you must pass in the plugin via client options. ```typescript import * as Ably from 'ably'; -import LiveObjects from 'ably/liveobjects'; +import Objects from 'ably/objects'; const client = new Ably.Realtime({ ...options, - plugins: { LiveObjects }, + plugins: { Objects }, }); ``` -LiveObjects plugin also works with the [Modular variant](#modular-tree-shakable-variant) of the library. +Objects plugin also works with the [Modular variant](#modular-tree-shakable-variant) of the library. -Alternatively, you can load the LiveObjects plugin directly in your HTML using `script` tag (in case you can't use a package manager): +Alternatively, you can load the Objects plugin directly in your HTML using `script` tag (in case you can't use a package manager): ```html - + ``` -When loaded this way, the LiveObjects plugin will be available on the global object via the `AblyLiveObjectsPlugin` property, so you will need to pass it to the Ably instance as follows: +When loaded this way, the Objects plugin will be available on the global object via the `AblyObjectsPlugin` property, so you will need to pass it to the Ably instance as follows: ```typescript const client = new Ably.Realtime({ ...options, - plugins: { LiveObjects: AblyLiveObjectsPlugin }, + plugins: { Objects: AblyObjectsPlugin }, }); ``` -The LiveObjects plugin is developed as part of the Ably client library, so it is available for the same versions as the Ably client library itself. It also means that it follows the same semantic versioning rules as they were defined for [the Ably client library](#for-browsers). For example, to lock into a major or minor version of the LiveObjects plugin, you can specify a specific version number such as https://cdn.ably.com/lib/liveobjects.umd.min-2.js for all v2._ versions, or https://cdn.ably.com/lib/liveobjects.umd.min-2.4.js for all v2.4._ versions, or you can lock into a single release with https://cdn.ably.com/lib/liveobjects.umd.min-2.4.0.js. Note you can load the non-minified version by omitting `.min` from the URL such as https://cdn.ably.com/lib/liveobjects.umd-2.js. +The Objects plugin is developed as part of the Ably client library, so it is available for the same versions as the Ably client library itself. It also means that it follows the same semantic versioning rules as they were defined for [the Ably client library](#for-browsers). For example, to lock into a major or minor version of the Objects plugin, you can specify a specific version number such as https://cdn.ably.com/lib/objects.umd.min-2.js for all v2._ versions, or https://cdn.ably.com/lib/objects.umd.min-2.4.js for all v2.4._ versions, or you can lock into a single release with https://cdn.ably.com/lib/objects.umd.min-2.4.0.js. Note you can load the non-minified version by omitting `.min` from the URL such as https://cdn.ably.com/lib/objects.umd-2.js. For more information about the LiveObjects product, see the [Ably LiveObjects documentation](https://ably.com/docs/products/liveobjects). -#### State Channel Modes +#### Objects Channel Modes -To use the LiveObjects feature, clients must attach to a channel with the correct channel mode: +To use the Objects on a channel, clients must attach to a channel with the correct channel mode: -- `state_subscribe` - required to retrieve state of Live Objects for a channel -- `state_publish` - required to create new and modify existing Live Objects on a channel +- `object_subscribe` - required to retrieve Objects for a channel +- `object_publish` - required to create new and modify existing Objects on a channel ```typescript const client = new Ably.Realtime({ // authentication options ...options, - plugins: { LiveObjects }, + plugins: { Objects }, }); -const channelOptions = { modes: ['state_subscribe', 'state_publish'] }; -const channel = client.channels.get('my_live_objects_channel', channelOptions); -const liveObjects = channel.liveObjects; +const channelOptions = { modes: ['object_subscribe', 'object_publish'] }; +const channel = client.channels.get('my_objects_channel', channelOptions); +const objects = channel.objects; ``` -The authentication token must include corresponding capabilities for the client to interact with LiveObjects. +The authentication token must include corresponding capabilities for the client to interact with Objects. #### Getting the Root Object -The root object represents the top-level entry point for LiveObjects state within a channel. It gives access to all other nested Live Objects. +The root object represents the top-level entry point for Objects within a channel. It gives access to all other nested Live Objects. ```typescript -const root = await liveObjects.getRoot(); +const root = await objects.getRoot(); ``` -The root object is a `LiveMap` instance and serves as the starting point for storing and organizing LiveObjects state. +The root object is a `LiveMap` instance and serves as the starting point for storing and organizing Objects on a channel. #### Live Object Types LiveObjects currently supports two primary data structures; `LiveMap` and `LiveCounter`. -`LiveMap` - A key/value map data structure, similar to a JavaScript `Map`, where all changes are synchronized across clients in realtime. It allows you to store primitive values and other Live Objects, enabling a composable state. +`LiveMap` - A key/value map data structure, similar to a JavaScript `Map`, where all changes are synchronized across clients in realtime. It allows you to store primitive values and other Live Objects, enabling composability. You can use `LiveMap` as follows: ```typescript // root object is a LiveMap -const root = await liveObjects.getRoot(); +const root = await objects.getRoot(); // you can read values for a key with .get root.get('foo'); @@ -690,7 +690,7 @@ await root.set('bar', 1); await root.set('baz', true); await root.set('qux', new Uint8Array([21, 31])); // as well as other live objects -const counter = await liveObjects.createCounter(); +const counter = await objects.createCounter(); await root.set('quux', counter); // and you can remove keys with .remove @@ -702,7 +702,7 @@ await root.remove('name'); You can use `LiveCounter` as follows: ```typescript -const counter = await liveObjects.createCounter(); +const counter = await objects.createCounter(); // you can get current value of a counter with .value counter.value(); @@ -714,14 +714,14 @@ await counter.decrement(2); #### Subscribing to Updates -Subscribing to updates on Live Objects allows you to receive changes made by other clients in realtime. Since multiple clients may modify the same Live Objects state, subscribing ensures that your application reacts to external updates as soon as they are received. +Subscribing to updates on Live Objects allows you to receive changes made by other clients in realtime. Since multiple clients may modify the same Live Objects, subscribing ensures that your application reacts to external updates as soon as they are received. Additionally, mutation methods such as `LiveMap.set`, `LiveCounter.increment`, and `LiveCounter.decrement` do not directly edit the current state of the object locally. Instead, they send the intended operation to the Ably system, and the change is applied to the local object only when the corresponding realtime operation is echoed back to the client. This means that the state you retrieve immediately after a mutation may not reflect the latest updates yet. You can subscribe to updates on all Live Objects using subscription listeners as follows: ```typescript -const root = await liveObjects.getRoot(); +const root = await objects.getRoot(); // subscribe to updates on a LiveMap root.subscribe((update: LiveMapUpdate) => { @@ -730,7 +730,7 @@ root.subscribe((update: LiveMapUpdate) => { }); // subscribe to updates on a LiveCounter -const counter = await liveObjects.createCounter(); +const counter = await objects.createCounter(); counter.subscribe((update: LiveCounterUpdate) => { console.log('LiveCounter new value:', counter.value()); // can read the current value of the counter inside this callback console.log('LiveCounter update details:', update); // and can get update details from the provided update object @@ -775,18 +775,18 @@ root.unsubscribeAll(); New `LiveMap` and `LiveCounter` instances can be created as follows: ```typescript -const counter = await liveObjects.createCounter(123); // with optional initial counter value -const map = await liveObjects.createMap({ key: 'value' }); // with optional initial map entries +const counter = await objects.createCounter(123); // with optional initial counter value +const map = await objects.createMap({ key: 'value' }); // with optional initial map entries ``` -To persist them within the LiveObjects state, they must be assigned to a parent `LiveMap` that is connected to the root object through the object hierarchy: +To persist them within the Objects state, they must be assigned to a parent `LiveMap` that is connected to the root object through the object hierarchy: ```typescript -const root = await liveObjects.getRoot(); +const root = await objects.getRoot(); -const counter = await liveObjects.createCounter(); -const map = await liveObjects.createMap({ counter }); -const outerMap = await liveObjects.createMap({ map }); +const counter = await objects.createCounter(); +const map = await objects.createMap({ counter }); +const outerMap = await objects.createMap({ map }); await root.set('outerMap', outerMap); @@ -804,7 +804,7 @@ Batching allows multiple operations to be grouped into a single message that is Within a batch callback, the `BatchContext` instance provides wrapper objects around regular Live Objects with a synchronous API for storing changes in the batch context. ```typescript -await liveObjects.batch((ctx) => { +await objects.batch((ctx) => { const root = ctx.getRoot(); root.set('foo', 'bar'); @@ -821,16 +821,16 @@ await liveObjects.batch((ctx) => { Live Objects emit lifecycle events to signal critical state changes, such as synchronization progress and object deletions. -**Synchronization Events** - the `syncing` and `synced` events notify when the LiveObjects state is being synchronized with the Ably service. These events can be useful for displaying loading indicators, preventing user edits during synchronization, or triggering application logic when the data was loaded for the first time. +**Synchronization Events** - the `syncing` and `synced` events notify when the Objects state is being synchronized with the Ably service. These events can be useful for displaying loading indicators, preventing user edits during synchronization, or triggering application logic when the data was loaded for the first time. ```typescript -liveObjects.on('syncing', () => { - console.log('LiveObjects state is syncing...'); +objects.on('syncing', () => { + console.log('Objects are syncing...'); // Example: Show a loading indicator }); -liveObjects.on('synced', () => { - console.log('LiveObjects state has been synced.'); +objects.on('synced', () => { + console.log('Objects have been synced.'); // Example: Hide loading indicator }); ``` @@ -838,7 +838,7 @@ liveObjects.on('synced', () => { **Object Deletion Events** - objects that have been orphaned for a long period (i.e., not connected to the state tree graph by being set as a key in a map accessible from the root map object) will eventually be deleted. Once a Live Object is deleted, it can no longer be interacted with. You should avoid accessing its data or trying to update its value and you should remove all references to the deleted object in your application. ```typescript -const root = await liveObjects.getRoot(); +const root = await objects.getRoot(); const counter = root.get('counter'); counter.on('deleted', () => { @@ -850,23 +850,23 @@ counter.on('deleted', () => { To unsubscribe from lifecycle events: ```typescript -// same API for channel.liveObjects and LiveObject instances +// same API for channel.objects and LiveObject instances // use dedicated off function from the .on call -const { off } = liveObjects.on('synced', () => {}); +const { off } = objects.on('synced', () => {}); off(); // call .off with an event name and a listener reference const listener = () => {}; -liveObjects.on('synced', listener); -liveObjects.off('synced', listener); +objects.on('synced', listener); +objects.off('synced', listener); // deregister all listeners using .offAll -liveObjects.offAll(); +objects.offAll(); ``` -#### Typing LiveObjects +#### Typing Objects -You can provide your own TypeScript typings for LiveObjects by providing a globally defined `LiveObjectsTypes` interface. +You can provide your own TypeScript typings for Objects by providing a globally defined `ObjectsTypes` interface. ```typescript // file: ably.config.d.ts @@ -880,23 +880,23 @@ type MyCustomRoot = { }; declare global { - export interface LiveObjectsTypes { + export interface ObjectsTypes { root: MyCustomRoot; } } ``` -This will enable code completion and editor hints when interacting with the LiveObjects API: +This will enable code completion and editor hints when interacting with the Objects API: ```typescript -const root = await liveObjects.getRoot(); // uses types defined by global LiveObjectsTypes interface by default +const root = await objects.getRoot(); // uses types defined by global ObjectsTypes interface by default const map = root.get('map'); // LiveMap<{ foo: string; counter: LiveCounter }> map.set('foo', 1); // TypeError map.get('counter').value(); // autocompletion for counter method names ``` -You can also provide typings for the LiveObjects state tree when calling the `liveObjects.getRoot` method, allowing you to have different state typings for different channels: +You can also provide typings for the channel Objects when calling the `objects.getRoot` method, allowing you to have different typings for different channels: ```typescript type ReactionsRoot = { @@ -908,8 +908,8 @@ type PollsRoot = { currentPoll: LiveMap; }; -const reactionsRoot = await reactionsChannel.liveObjects.getRoot(); -const pollsRoot = await pollsChannel.liveObjects.getRoot(); +const reactionsRoot = await reactionsChannel.objects.getRoot(); +const pollsRoot = await pollsChannel.objects.getRoot(); ``` ## Delta Plugin diff --git a/ably.d.ts b/ably.d.ts index ca2119baf5..a2b44643ea 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -625,9 +625,9 @@ export interface CorePlugins { Push?: unknown; /** - * A plugin which allows the client to use LiveObjects functionality at {@link RealtimeChannel.liveObjects}. + * A plugin which allows the client to use LiveObjects functionality at {@link RealtimeChannel.objects}. */ - LiveObjects?: unknown; + Objects?: unknown; } /** @@ -872,13 +872,13 @@ declare namespace ChannelModes { */ type PRESENCE_SUBSCRIBE = 'PRESENCE_SUBSCRIBE'; /** - * The client can publish LiveObjects state messages. + * The client can publish Objects messages. */ - type STATE_PUBLISH = 'STATE_PUBLISH'; + type OBJECT_PUBLISH = 'OBJECT_PUBLISH'; /** - * The client can receive LiveObjects state messages. + * The client can receive Objects messages. */ - type STATE_SUBSCRIBE = 'STATE_SUBSCRIBE'; + type OBJECT_SUBSCRIBE = 'OBJECT_SUBSCRIBE'; /** * The client is resuming an existing connection. */ @@ -893,8 +893,8 @@ export type ChannelMode = | ChannelModes.SUBSCRIBE | ChannelModes.PRESENCE | ChannelModes.PRESENCE_SUBSCRIBE - | ChannelModes.STATE_PUBLISH - | ChannelModes.STATE_SUBSCRIBE + | ChannelModes.OBJECT_PUBLISH + | ChannelModes.OBJECT_SUBSCRIBE | ChannelModes.ATTACH_RESUME; /** @@ -1567,9 +1567,9 @@ export type ErrorCallback = (error: ErrorInfo | null) => void; export type LiveObjectUpdateCallback = (update: T) => void; /** - * The callback used for the events emitted by {@link LiveObjects}. + * The callback used for the events emitted by {@link Objects}. */ -export type LiveObjectsEventCallback = () => void; +export type ObjectsEventCallback = () => void; /** * The callback used for the lifecycle events emitted by {@link LiveObject}. @@ -1577,11 +1577,11 @@ export type LiveObjectsEventCallback = () => void; export type LiveObjectLifecycleEventCallback = () => void; /** - * A function passed to {@link LiveObjects.batch} to group multiple Live Object operations into a single channel message. + * A function passed to {@link Objects.batch} to group multiple Objects operations into a single channel message. * * Must not be `async`. * - * @param batchContext - A {@link BatchContext} object that allows grouping operations on Live Objects for this batch. + * @param batchContext - A {@link BatchContext} object that allows grouping Objects operations for this batch. */ export type BatchCallback = (batchContext: BatchContext) => void; @@ -2052,30 +2052,30 @@ export declare interface PushChannel { } /** - * The `LiveObjectsEvents` namespace describes the possible values of the {@link LiveObjectsEvent} type. + * The `ObjectsEvents` namespace describes the possible values of the {@link ObjectsEvent} type. */ -declare namespace LiveObjectsEvents { +declare namespace ObjectsEvents { /** - * The local LiveObjects state is currently being synchronized with the Ably service. + * The local copy of Objects on a channel is currently being synchronized with the Ably service. */ type SYNCING = 'syncing'; /** - * The local LiveObjects state has been synchronized with the Ably service. + * The local copy of Objects on a channel has been synchronized with the Ably service. */ type SYNCED = 'synced'; } /** - * Describes the events emitted by a {@link LiveObjects} object. + * Describes the events emitted by a {@link Objects} object. */ -export type LiveObjectsEvent = LiveObjectsEvents.SYNCED | LiveObjectsEvents.SYNCING; +export type ObjectsEvent = ObjectsEvents.SYNCED | ObjectsEvents.SYNCING; /** * The `LiveObjectLifecycleEvents` namespace describes the possible values of the {@link LiveObjectLifecycleEvent} type. */ declare namespace LiveObjectLifecycleEvents { /** - * Indicates that the object has been deleted from the LiveObjects pool and should no longer be interacted with. + * Indicates that the object has been deleted from the Objects pool and should no longer be interacted with. */ type DELETED = 'deleted'; } @@ -2086,15 +2086,15 @@ declare namespace LiveObjectLifecycleEvents { export type LiveObjectLifecycleEvent = LiveObjectLifecycleEvents.DELETED; /** - * Enables the LiveObjects state to be subscribed to for a channel. + * Enables the Objects to be read, modified and subscribed to for a channel. */ -export declare interface LiveObjects { +export declare interface Objects { /** - * Retrieves the root {@link LiveMap} object for state on a channel. + * Retrieves the root {@link LiveMap} object for Objects on a channel. * - * A type parameter can be provided to describe the structure of the LiveObjects state on the channel. By default, it uses types from the globally defined `LiveObjectsTypes` interface. + * A type parameter can be provided to describe the structure of the Objects on the channel. By default, it uses types from the globally defined `ObjectsTypes` interface. * - * You can specify custom types for LiveObjects by defining a global `LiveObjectsTypes` interface with a `root` property that conforms to {@link LiveMapType}. + * You can specify custom types for Objects by defining a global `ObjectsTypes` interface with a `root` property that conforms to {@link LiveMapType}. * * Example: * @@ -2106,7 +2106,7 @@ export declare interface LiveObjects { * }; * * declare global { - * export interface LiveObjectsTypes { + * export interface ObjectsTypes { * root: MyRoot; * } * } @@ -2137,7 +2137,7 @@ export declare interface LiveObjects { * As a result, other clients will receive the changes as a single channel message after the batch function has completed. * * This method accepts a synchronous callback, which is provided with a {@link BatchContext} object. - * Use the context object to access Live Object instances in your state and batch operations for them. + * Use the context object to access Objects on a channel and batch operations for them. * * The objects' data is not modified inside the callback function. Instead, the objects will be updated * when the batched operations are applied by the Ably service and echoed back to the client. @@ -2152,9 +2152,9 @@ export declare interface LiveObjects { * * @param event - The named event to listen for. * @param callback - The event listener. - * @returns A {@link OnLiveObjectsEventResponse} object that allows the provided listener to be deregistered from future updates. + * @returns A {@link OnObjectsEventResponse} object that allows the provided listener to be deregistered from future updates. */ - on(event: LiveObjectsEvent, callback: LiveObjectsEventCallback): OnLiveObjectsEventResponse; + on(event: ObjectsEvent, callback: ObjectsEventCallback): OnObjectsEventResponse; /** * Removes all registrations that match both the specified listener and the specified event. @@ -2162,7 +2162,7 @@ export declare interface LiveObjects { * @param event - The named event. * @param callback - The event listener. */ - off(event: LiveObjectsEvent, callback: LiveObjectsEventCallback): void; + off(event: ObjectsEvent, callback: ObjectsEventCallback): void; /** * Deregisters all registrations, for all events and listeners. @@ -2172,39 +2172,39 @@ export declare interface LiveObjects { declare global { /** - * A globally defined interface that allows users to define custom types for LiveObjects. + * A globally defined interface that allows users to define custom types for Objects. */ - export interface LiveObjectsTypes { + export interface ObjectsTypes { [key: string]: unknown; } } /** * Represents the type of data stored in a {@link LiveMap}. - * It maps string keys to scalar values ({@link StateValue}), or other LiveObjects. + * It maps string keys to scalar values ({@link StateValue}), or other Live Objects. */ export type LiveMapType = { [key: string]: StateValue | LiveMap | LiveCounter | undefined }; /** - * The default type for the `root` object in the LiveObjects, based on the globally defined {@link LiveObjectsTypes} interface. + * The default type for the `root` object for Objects on a channel, based on the globally defined {@link ObjectsTypes} interface. * - * - If no custom types are provided in `LiveObjectsTypes`, defaults to an untyped root map representation using the {@link LiveMapType} interface. - * - If a `root` type exists in `LiveObjectsTypes` and conforms to the {@link LiveMapType} interface, it is used as the type for the `root` object. + * - If no custom types are provided in `ObjectsTypes`, defaults to an untyped root map representation using the {@link LiveMapType} interface. + * - If a `root` type exists in `ObjectsTypes` and conforms to the {@link LiveMapType} interface, it is used as the type for the `root` object. * - If the provided `root` type does not match {@link LiveMapType}, a type error message is returned. */ export type DefaultRoot = // we need a way to know when no types were provided by the user. - // we expect a "root" property to be set on LiveObjectsTypes interface, e.g. it won't be "unknown" anymore - unknown extends LiveObjectsTypes['root'] + // we expect a "root" property to be set on ObjectsTypes interface, e.g. it won't be "unknown" anymore + unknown extends ObjectsTypes['root'] ? LiveMapType // no custom types provided; use the default untyped map representation for the root - : LiveObjectsTypes['root'] extends LiveMapType - ? LiveObjectsTypes['root'] // "root" property exists, and it is of an expected type, we can use this interface for the root object in LiveObjects. - : `Provided type definition for the "root" object in LiveObjectsTypes is not of an expected LiveMapType`; + : ObjectsTypes['root'] extends LiveMapType + ? ObjectsTypes['root'] // "root" property exists, and it is of an expected type, we can use this interface for the root object in Objects. + : `Provided type definition for the "root" object in ObjectsTypes is not of an expected LiveMapType`; /** * Object returned from an `on` call, allowing the listener provided in that call to be deregistered. */ -export declare interface OnLiveObjectsEventResponse { +export declare interface OnObjectsEventResponse { /** * Deregisters the listener passed to the `on` call. */ @@ -2212,11 +2212,11 @@ export declare interface OnLiveObjectsEventResponse { } /** - * Enables grouping multiple Live Object operations together by providing `BatchContext*` wrapper objects. + * Enables grouping multiple Objects operations together by providing `BatchContext*` wrapper objects. */ export declare interface BatchContext { /** - * Mirrors the {@link LiveObjects.getRoot} method and returns a {@link BatchContextLiveMap} wrapper for the root object on a channel. + * Mirrors the {@link Objects.getRoot} method and returns a {@link BatchContextLiveMap} wrapper for the root object on a channel. * * @returns A {@link BatchContextLiveMap} object. */ @@ -2420,7 +2420,7 @@ export declare interface LiveCounterUpdate extends LiveObjectUpdate { } /** - * Describes the common interface for all conflict-free data structures supported by the `LiveObjects`. + * Describes the common interface for all conflict-free data structures supported by the Objects. */ export declare interface LiveObject { /** @@ -2626,9 +2626,9 @@ export declare interface RealtimeChannel extends EventEmitter = { +const buildablePlugins: Record<'push' | 'objects', PluginInfo> = { push: { description: 'Push', path: './build/push.js', external: ['ulid'] }, - liveObjects: { description: 'LiveObjects', path: './build/liveobjects.js', external: ['deep-equal'] }, + objects: { description: 'Objects', path: './build/objects.js', external: ['deep-equal'] }, }; function formatBytes(bytes: number) { @@ -215,8 +215,8 @@ async function calculatePushPluginSize(): Promise { return calculatePluginSize(buildablePlugins.push); } -async function calculateLiveObjectsPluginSize(): Promise { - return calculatePluginSize(buildablePlugins.liveObjects); +async function calculateObjectsPluginSize(): Promise { + return calculatePluginSize(buildablePlugins.objects); } async function calculateAndCheckMinimalUsefulRealtimeBundleSize(): Promise { @@ -317,11 +317,11 @@ async function checkPushPluginFiles() { return checkBundleFiles(pushPluginBundleInfo, allowedFiles, 100); } -async function checkLiveObjectsPluginFiles() { - const { path, external } = buildablePlugins.liveObjects; +async function checkObjectsPluginFiles() { + const { path, external } = buildablePlugins.objects; const pluginBundleInfo = getBundleInfo(path, undefined, external); - // These are the files that are allowed to contribute >= `threshold` bytes to the LiveObjects bundle. + // These are the files that are allowed to contribute >= `threshold` bytes to the Objects bundle. const allowedFiles = new Set([ 'src/plugins/liveobjects/batchcontext.ts', 'src/plugins/liveobjects/batchcontextlivecounter.ts', @@ -391,7 +391,7 @@ async function checkBundleFiles(bundleInfo: BundleInfo, allowedFiles: Set ({ tableRows: [...accum.tableRows, ...current.tableRows], @@ -400,7 +400,7 @@ async function checkBundleFiles(bundleInfo: BundleInfo, allowedFiles: Set - this.client._LiveObjectsPlugin - ? this.client._LiveObjectsPlugin.StateMessage.decode(msg, options, MessageEncoding) - : Utils.throwMissingPluginError('LiveObjects'), + this.client._objectsPlugin + ? this.client._objectsPlugin.StateMessage.decode(msg, options, MessageEncoding) + : Utils.throwMissingPluginError('Objects'), ); if (message.action === actions.STATE) { - this._liveObjects.handleStateMessages(stateMessages); + this._objects.handleStateMessages(stateMessages); } else { - this._liveObjects.handleStateSyncMessages(stateMessages, message.channelSerial); + this._objects.handleStateSyncMessages(stateMessages, message.channelSerial); } break; @@ -830,8 +830,8 @@ class RealtimeChannel extends EventEmitter { if (this._presence) { this._presence.actOnChannelState(state, hasPresence, reason); } - if (this._liveObjects) { - this._liveObjects.actOnChannelState(state, hasState); + if (this._objects) { + this._objects.actOnChannelState(state, hasState); } if (state === 'suspended' && this.connectionManager.state.sendEvents) { this.startRetryTimer(); diff --git a/src/common/lib/transport/comettransport.ts b/src/common/lib/transport/comettransport.ts index 00ecb804d8..c63462b272 100644 --- a/src/common/lib/transport/comettransport.ts +++ b/src/common/lib/transport/comettransport.ts @@ -356,7 +356,7 @@ abstract class CometTransport extends Transport { protocolMessageFromDeserialized( items[i], this.connectionManager.realtime._RealtimePresence, - this.connectionManager.realtime._LiveObjectsPlugin, + this.connectionManager.realtime._objectsPlugin, ), ); } catch (e) { diff --git a/src/common/lib/transport/connectionmanager.ts b/src/common/lib/transport/connectionmanager.ts index 56364d6c95..2dabd48dab 100644 --- a/src/common/lib/transport/connectionmanager.ts +++ b/src/common/lib/transport/connectionmanager.ts @@ -1805,8 +1805,7 @@ class ConnectionManager extends EventEmitter { Logger.LOG_MICRO, 'ConnectionManager.send()', - 'queueing msg; ' + - stringifyProtocolMessage(msg, this.realtime._RealtimePresence, this.realtime._LiveObjectsPlugin), + 'queueing msg; ' + stringifyProtocolMessage(msg, this.realtime._RealtimePresence, this.realtime._objectsPlugin), ); } this.queue(msg, callback); diff --git a/src/common/lib/transport/protocol.ts b/src/common/lib/transport/protocol.ts index 47a71171ff..05d50da966 100644 --- a/src/common/lib/transport/protocol.ts +++ b/src/common/lib/transport/protocol.ts @@ -81,7 +81,7 @@ class Protocol extends EventEmitter { stringifyProtocolMessage( pendingMessage.message, this.transport.connectionManager.realtime._RealtimePresence, - this.transport.connectionManager.realtime._LiveObjectsPlugin, + this.transport.connectionManager.realtime._objectsPlugin, ), ); } diff --git a/src/common/lib/transport/transport.ts b/src/common/lib/transport/transport.ts index 3f298e24f5..84cf19226d 100644 --- a/src/common/lib/transport/transport.ts +++ b/src/common/lib/transport/transport.ts @@ -131,7 +131,7 @@ abstract class Transport extends EventEmitter { stringifyProtocolMessage( message, this.connectionManager.realtime._RealtimePresence, - this.connectionManager.realtime._LiveObjectsPlugin, + this.connectionManager.realtime._objectsPlugin, ) + '; connectionId = ' + this.connectionManager.connectionId, diff --git a/src/common/lib/transport/websockettransport.ts b/src/common/lib/transport/websockettransport.ts index a85a6f077e..cefc061dbd 100644 --- a/src/common/lib/transport/websockettransport.ts +++ b/src/common/lib/transport/websockettransport.ts @@ -140,7 +140,7 @@ class WebSocketTransport extends Transport { data, this.connectionManager.realtime._MsgPack, this.connectionManager.realtime._RealtimePresence, - this.connectionManager.realtime._LiveObjectsPlugin, + this.connectionManager.realtime._objectsPlugin, this.format, ), ); diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index fb473177aa..1750e14346 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -12,7 +12,7 @@ import PresenceMessage, { fromValues as presenceMessageFromValues, fromValuesArray as presenceMessagesFromValuesArray, } from './presencemessage'; -import type * as LiveObjectsPlugin from 'plugins/liveobjects'; +import type * as ObjectsPlugin from 'plugins/liveobjects'; export const actions = { HEARTBEAT: 0, @@ -55,8 +55,8 @@ const flags: { [key: string]: number } = { PUBLISH: 1 << 17, SUBSCRIBE: 1 << 18, PRESENCE_SUBSCRIBE: 1 << 19, - STATE_SUBSCRIBE: 1 << 24, - STATE_PUBLISH: 1 << 25, + OBJECT_SUBSCRIBE: 1 << 24, + OBJECT_PUBLISH: 1 << 25, HAS_STATE: 1 << 26, }; const flagNames = Object.keys(flags); @@ -65,8 +65,8 @@ flags.MODE_ALL = flags.PUBLISH | flags.SUBSCRIBE | flags.PRESENCE_SUBSCRIBE | - flags.STATE_SUBSCRIBE | - flags.STATE_PUBLISH; + flags.OBJECT_SUBSCRIBE | + flags.OBJECT_PUBLISH; function toStringArray(array?: any[]): string { const result = []; @@ -83,8 +83,8 @@ export const channelModes = [ 'PUBLISH', 'SUBSCRIBE', 'PRESENCE_SUBSCRIBE', - 'STATE_SUBSCRIBE', - 'STATE_PUBLISH', + 'OBJECT_SUBSCRIBE', + 'OBJECT_PUBLISH', ]; export const serialize = Utils.encodeBody; @@ -93,17 +93,17 @@ export function deserialize( serialized: unknown, MsgPack: MsgPack | null, presenceMessagePlugin: PresenceMessagePlugin | null, - liveObjectsPlugin: typeof LiveObjectsPlugin | null, + objectsPlugin: typeof ObjectsPlugin | null, format?: Utils.Format, ): ProtocolMessage { const deserialized = Utils.decodeBody>(serialized, MsgPack, format); - return fromDeserialized(deserialized, presenceMessagePlugin, liveObjectsPlugin); + return fromDeserialized(deserialized, presenceMessagePlugin, objectsPlugin); } export function fromDeserialized( deserialized: Record, presenceMessagePlugin: PresenceMessagePlugin | null, - liveObjectsPlugin: typeof LiveObjectsPlugin | null, + objectsPlugin: typeof ObjectsPlugin | null, ): ProtocolMessage { const error = deserialized.error; if (error) { @@ -126,12 +126,12 @@ export function fromDeserialized( } } - let state: LiveObjectsPlugin.StateMessage[] | undefined = undefined; - if (liveObjectsPlugin) { - state = deserialized.state as LiveObjectsPlugin.StateMessage[]; + let state: ObjectsPlugin.StateMessage[] | undefined = undefined; + if (objectsPlugin) { + state = deserialized.state as ObjectsPlugin.StateMessage[]; if (state) { for (let i = 0; i < state.length; i++) { - state[i] = liveObjectsPlugin.StateMessage.fromValues(state[i], Utils, MessageEncoding); + state[i] = objectsPlugin.StateMessage.fromValues(state[i], Utils, MessageEncoding); } } } @@ -142,17 +142,15 @@ export function fromDeserialized( /** * Used internally by the tests. * - * LiveObjectsPlugin code can't be included as part of the core library to prevent size growth, - * so if a test needs to build Live Object state messages, then it must provide LiveObjectsPlugin. + * ObjectsPlugin code can't be included as part of the core library to prevent size growth, + * so if a test needs to build state messages, then it must provide the plugin upon call. */ -export function makeFromDeserializedWithDependencies(dependencies?: { - LiveObjectsPlugin: typeof LiveObjectsPlugin | null; -}) { +export function makeFromDeserializedWithDependencies(dependencies?: { ObjectsPlugin: typeof ObjectsPlugin | null }) { return (deserialized: Record): ProtocolMessage => { return fromDeserialized( deserialized, { presenceMessageFromValues, presenceMessagesFromValuesArray }, - dependencies?.LiveObjectsPlugin ?? null, + dependencies?.ObjectsPlugin ?? null, ); }; } @@ -164,7 +162,7 @@ export function fromValues(values: unknown): ProtocolMessage { export function stringify( msg: any, presenceMessagePlugin: PresenceMessagePlugin | null, - liveObjectsPlugin: typeof LiveObjectsPlugin | null, + objectsPlugin: typeof ObjectsPlugin | null, ): string { let result = '[ProtocolMessage'; if (msg.action !== undefined) result += '; action=' + ActionName[msg.action] || msg.action; @@ -179,9 +177,8 @@ export function stringify( if (msg.messages) result += '; messages=' + toStringArray(messagesFromValuesArray(msg.messages)); if (msg.presence && presenceMessagePlugin) result += '; presence=' + toStringArray(presenceMessagePlugin.presenceMessagesFromValuesArray(msg.presence)); - if (msg.state && liveObjectsPlugin) { - result += - '; state=' + toStringArray(liveObjectsPlugin.StateMessage.fromValuesArray(msg.state, Utils, MessageEncoding)); + if (msg.state && objectsPlugin) { + result += '; state=' + toStringArray(objectsPlugin.StateMessage.fromValuesArray(msg.state, Utils, MessageEncoding)); } if (msg.error) result += '; error=' + ErrorInfo.fromValues(msg.error).toString(); if (msg.auth && msg.auth.accessToken) result += '; token=' + msg.auth.accessToken; @@ -219,9 +216,9 @@ class ProtocolMessage { */ presence?: PresenceMessage[]; /** - * This will be undefined if we skipped decoding this property due to user not requesting LiveObjects functionality — see {@link fromDeserialized} + * This will be undefined if we skipped decoding this property due to user not requesting Objects functionality — see {@link fromDeserialized} */ - state?: LiveObjectsPlugin.StateMessage[]; + state?: ObjectsPlugin.StateMessage[]; auth?: unknown; connectionDetails?: Record; diff --git a/src/plugins/index.d.ts b/src/plugins/index.d.ts index 035cc3d14e..236cc9d1d1 100644 --- a/src/plugins/index.d.ts +++ b/src/plugins/index.d.ts @@ -1,7 +1,7 @@ -import LiveObjects from './liveobjects'; +import Objects from './liveobjects'; import Push from './push'; export interface StandardPlugins { - LiveObjects?: typeof LiveObjects; + Objects?: typeof Objects; Push?: typeof Push; } diff --git a/src/plugins/liveobjects/batchcontext.ts b/src/plugins/liveobjects/batchcontext.ts index 1d2f6aef72..5fad94d6df 100644 --- a/src/plugins/liveobjects/batchcontext.ts +++ b/src/plugins/liveobjects/batchcontext.ts @@ -4,27 +4,27 @@ import { BatchContextLiveCounter } from './batchcontextlivecounter'; import { BatchContextLiveMap } from './batchcontextlivemap'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; -import { LiveObjects } from './liveobjects'; +import { Objects } from './liveobjects'; import { ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage } from './statemessage'; export class BatchContext { private _client: BaseClient; - /** Maps object ids to the corresponding batch context object wrappers for Live Objects in the pool */ + /** Maps live object ids to the corresponding batch context object wrappers */ private _wrappedObjects: Map> = new Map(); private _queuedMessages: StateMessage[] = []; private _isClosed = false; constructor( - private _liveObjects: LiveObjects, + private _objects: Objects, private _root: LiveMap, ) { - this._client = _liveObjects.getClient(); - this._wrappedObjects.set(this._root.getObjectId(), new BatchContextLiveMap(this, this._liveObjects, this._root)); + this._client = _objects.getClient(); + this._wrappedObjects.set(this._root.getObjectId(), new BatchContextLiveMap(this, this._objects, this._root)); } getRoot(): BatchContextLiveMap { - this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._objects.throwIfInvalidAccessApiConfiguration(); this.throwIfClosed(); return this.getWrappedObject(ROOT_OBJECT_ID) as BatchContextLiveMap; } @@ -37,16 +37,16 @@ export class BatchContext { return this._wrappedObjects.get(objectId); } - const originObject = this._liveObjects.getPool().get(objectId); + const originObject = this._objects.getPool().get(objectId); if (!originObject) { return undefined; } let wrappedObject: BatchContextLiveCounter | BatchContextLiveMap; if (originObject instanceof LiveMap) { - wrappedObject = new BatchContextLiveMap(this, this._liveObjects, originObject); + wrappedObject = new BatchContextLiveMap(this, this._objects, originObject); } else if (originObject instanceof LiveCounter) { - wrappedObject = new BatchContextLiveCounter(this, this._liveObjects, originObject); + wrappedObject = new BatchContextLiveCounter(this, this._objects, originObject); } else { throw new this._client.ErrorInfo( `Unknown Live Object instance type: objectId=${originObject.getObjectId()}`, @@ -97,7 +97,7 @@ export class BatchContext { this.close(); if (this._queuedMessages.length > 0) { - await this._liveObjects.publish(this._queuedMessages); + await this._objects.publish(this._queuedMessages); } } finally { this._wrappedObjects.clear(); diff --git a/src/plugins/liveobjects/batchcontextlivecounter.ts b/src/plugins/liveobjects/batchcontextlivecounter.ts index c2a673f958..63ecaee09b 100644 --- a/src/plugins/liveobjects/batchcontextlivecounter.ts +++ b/src/plugins/liveobjects/batchcontextlivecounter.ts @@ -1,34 +1,34 @@ import type BaseClient from 'common/lib/client/baseclient'; import { BatchContext } from './batchcontext'; import { LiveCounter } from './livecounter'; -import { LiveObjects } from './liveobjects'; +import { Objects } from './liveobjects'; export class BatchContextLiveCounter { private _client: BaseClient; constructor( private _batchContext: BatchContext, - private _liveObjects: LiveObjects, + private _objects: Objects, private _counter: LiveCounter, ) { - this._client = this._liveObjects.getClient(); + this._client = this._objects.getClient(); } value(): number { - this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._objects.throwIfInvalidAccessApiConfiguration(); this._batchContext.throwIfClosed(); return this._counter.value(); } increment(amount: number): void { - this._liveObjects.throwIfInvalidWriteApiConfiguration(); + this._objects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); - const stateMessage = LiveCounter.createCounterIncMessage(this._liveObjects, this._counter.getObjectId(), amount); + const stateMessage = LiveCounter.createCounterIncMessage(this._objects, this._counter.getObjectId(), amount); this._batchContext.queueStateMessage(stateMessage); } decrement(amount: number): void { - this._liveObjects.throwIfInvalidWriteApiConfiguration(); + this._objects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); // do an explicit type safety check here before negating the amount value, // so we don't unintentionally change the type sent by a user diff --git a/src/plugins/liveobjects/batchcontextlivemap.ts b/src/plugins/liveobjects/batchcontextlivemap.ts index 2e88b260f8..7971b17555 100644 --- a/src/plugins/liveobjects/batchcontextlivemap.ts +++ b/src/plugins/liveobjects/batchcontextlivemap.ts @@ -2,17 +2,17 @@ import type * as API from '../../../ably'; import { BatchContext } from './batchcontext'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; -import { LiveObjects } from './liveobjects'; +import { Objects } from './liveobjects'; export class BatchContextLiveMap { constructor( private _batchContext: BatchContext, - private _liveObjects: LiveObjects, + private _objects: Objects, private _map: LiveMap, ) {} get(key: TKey): T[TKey] | undefined { - this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._objects.throwIfInvalidAccessApiConfiguration(); this._batchContext.throwIfClosed(); const value = this._map.get(key); if (value instanceof LiveObject) { @@ -23,40 +23,40 @@ export class BatchContextLiveMap { } size(): number { - this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._objects.throwIfInvalidAccessApiConfiguration(); this._batchContext.throwIfClosed(); return this._map.size(); } *entries(): IterableIterator<[TKey, T[TKey]]> { - this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._objects.throwIfInvalidAccessApiConfiguration(); this._batchContext.throwIfClosed(); yield* this._map.entries(); } *keys(): IterableIterator { - this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._objects.throwIfInvalidAccessApiConfiguration(); this._batchContext.throwIfClosed(); yield* this._map.keys(); } *values(): IterableIterator { - this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._objects.throwIfInvalidAccessApiConfiguration(); this._batchContext.throwIfClosed(); yield* this._map.values(); } set(key: TKey, value: T[TKey]): void { - this._liveObjects.throwIfInvalidWriteApiConfiguration(); + this._objects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); - const stateMessage = LiveMap.createMapSetMessage(this._liveObjects, this._map.getObjectId(), key, value); + const stateMessage = LiveMap.createMapSetMessage(this._objects, this._map.getObjectId(), key, value); this._batchContext.queueStateMessage(stateMessage); } remove(key: TKey): void { - this._liveObjects.throwIfInvalidWriteApiConfiguration(); + this._objects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); - const stateMessage = LiveMap.createMapRemoveMessage(this._liveObjects, this._map.getObjectId(), key); + const stateMessage = LiveMap.createMapRemoveMessage(this._objects, this._map.getObjectId(), key); this._batchContext.queueStateMessage(stateMessage); } } diff --git a/src/plugins/liveobjects/index.ts b/src/plugins/liveobjects/index.ts index 350024ae94..11ed1762b2 100644 --- a/src/plugins/liveobjects/index.ts +++ b/src/plugins/liveobjects/index.ts @@ -1,9 +1,9 @@ -import { LiveObjects } from './liveobjects'; +import { Objects } from './liveobjects'; import { StateMessage } from './statemessage'; -export { LiveObjects, StateMessage }; +export { Objects, StateMessage }; export default { - LiveObjects, + Objects, StateMessage, }; diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/liveobjects/livecounter.ts index 5762a8cdbb..216266e9fa 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/liveobjects/livecounter.ts @@ -1,5 +1,5 @@ import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; -import { LiveObjects } from './liveobjects'; +import { Objects } from './liveobjects'; import { ObjectId } from './objectid'; import { StateCounterOp, StateMessage, StateObject, StateOperation, StateOperationAction } from './statemessage'; @@ -17,8 +17,8 @@ export class LiveCounter extends LiveObject * * @internal */ - static zeroValue(liveobjects: LiveObjects, objectId: string): LiveCounter { - return new LiveCounter(liveobjects, objectId); + static zeroValue(objects: Objects, objectId: string): LiveCounter { + return new LiveCounter(objects, objectId); } /** @@ -27,8 +27,8 @@ export class LiveCounter extends LiveObject * * @internal */ - static fromStateObject(liveobjects: LiveObjects, stateObject: StateObject): LiveCounter { - const obj = new LiveCounter(liveobjects, stateObject.objectId); + static fromStateObject(objects: Objects, stateObject: StateObject): LiveCounter { + const obj = new LiveCounter(objects, stateObject.objectId); obj.overrideWithStateObject(stateObject); return obj; } @@ -39,8 +39,8 @@ export class LiveCounter extends LiveObject * * @internal */ - static fromStateOperation(liveobjects: LiveObjects, stateOperation: StateOperation): LiveCounter { - const obj = new LiveCounter(liveobjects, stateOperation.objectId); + static fromStateOperation(objects: Objects, stateOperation: StateOperation): LiveCounter { + const obj = new LiveCounter(objects, stateOperation.objectId); obj._mergeInitialDataFromCreateOperation(stateOperation); return obj; } @@ -48,8 +48,8 @@ export class LiveCounter extends LiveObject /** * @internal */ - static createCounterIncMessage(liveObjects: LiveObjects, objectId: string, amount: number): StateMessage { - const client = liveObjects.getClient(); + static createCounterIncMessage(objects: Objects, objectId: string, amount: number): StateMessage { + const client = objects.getClient(); if (typeof amount !== 'number' || !isFinite(amount)) { throw new client.ErrorInfo('Counter value increment should be a valid number', 40003, 400); @@ -73,8 +73,8 @@ export class LiveCounter extends LiveObject /** * @internal */ - static async createCounterCreateMessage(liveObjects: LiveObjects, count?: number): Promise { - const client = liveObjects.getClient(); + static async createCounterCreateMessage(objects: Objects, count?: number): Promise { + const client = objects.getClient(); if (count !== undefined && (typeof count !== 'number' || !Number.isFinite(count))) { throw new client.ErrorInfo('Counter value should be a valid number', 40003, 400); @@ -123,7 +123,7 @@ export class LiveCounter extends LiveObject } value(): number { - this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._objects.throwIfInvalidAccessApiConfiguration(); return this._dataRef.data; } @@ -137,16 +137,16 @@ export class LiveCounter extends LiveObject * @returns A promise which resolves upon receiving the ACK message for the published operation message. */ async increment(amount: number): Promise { - this._liveObjects.throwIfInvalidWriteApiConfiguration(); - const stateMessage = LiveCounter.createCounterIncMessage(this._liveObjects, this.getObjectId(), amount); - return this._liveObjects.publish([stateMessage]); + this._objects.throwIfInvalidWriteApiConfiguration(); + const stateMessage = LiveCounter.createCounterIncMessage(this._objects, this.getObjectId(), amount); + return this._objects.publish([stateMessage]); } /** * An alias for calling {@link LiveCounter.increment | LiveCounter.increment(-amount)} */ async decrement(amount: number): Promise { - this._liveObjects.throwIfInvalidWriteApiConfiguration(); + this._objects.throwIfInvalidWriteApiConfiguration(); // do an explicit type safety check here before negating the amount value, // so we don't unintentionally change the type sent by a user if (typeof amount !== 'number' || !isFinite(amount)) { diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/liveobjects/livemap.ts index 6b662fb0b2..11178f232c 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/liveobjects/livemap.ts @@ -3,7 +3,7 @@ import deepEqual from 'deep-equal'; import type * as API from '../../../ably'; import { DEFAULTS } from './defaults'; import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; -import { LiveObjects } from './liveobjects'; +import { Objects } from './liveobjects'; import { ObjectId } from './objectid'; import { MapSemantics, @@ -53,11 +53,11 @@ export interface LiveMapUpdate extends LiveObjectUpdate { export class LiveMap extends LiveObject { constructor( - liveObjects: LiveObjects, + objects: Objects, private _semantics: MapSemantics, objectId: string, ) { - super(liveObjects, objectId); + super(objects, objectId); } /** @@ -65,8 +65,8 @@ export class LiveMap extends LiveObject(liveobjects: LiveObjects, objectId: string): LiveMap { - return new LiveMap(liveobjects, MapSemantics.LWW, objectId); + static zeroValue(objects: Objects, objectId: string): LiveMap { + return new LiveMap(objects, MapSemantics.LWW, objectId); } /** @@ -75,8 +75,8 @@ export class LiveMap extends LiveObject(liveobjects: LiveObjects, stateObject: StateObject): LiveMap { - const obj = new LiveMap(liveobjects, stateObject.map?.semantics!, stateObject.objectId); + static fromStateObject(objects: Objects, stateObject: StateObject): LiveMap { + const obj = new LiveMap(objects, stateObject.map?.semantics!, stateObject.objectId); obj.overrideWithStateObject(stateObject); return obj; } @@ -87,11 +87,8 @@ export class LiveMap extends LiveObject( - liveobjects: LiveObjects, - stateOperation: StateOperation, - ): LiveMap { - const obj = new LiveMap(liveobjects, stateOperation.map?.semantics!, stateOperation.objectId); + static fromStateOperation(objects: Objects, stateOperation: StateOperation): LiveMap { + const obj = new LiveMap(objects, stateOperation.map?.semantics!, stateOperation.objectId); obj._mergeInitialDataFromCreateOperation(stateOperation); return obj; } @@ -100,14 +97,14 @@ export class LiveMap extends LiveObject( - liveObjects: LiveObjects, + objects: Objects, objectId: string, key: TKey, value: API.LiveMapType[TKey], ): StateMessage { - const client = liveObjects.getClient(); + const client = objects.getClient(); - LiveMap.validateKeyValue(liveObjects, key, value); + LiveMap.validateKeyValue(objects, key, value); const stateData: StateData = value instanceof LiveObject @@ -136,11 +133,11 @@ export class LiveMap extends LiveObject( - liveObjects: LiveObjects, + objects: Objects, objectId: string, key: TKey, ): StateMessage { - const client = liveObjects.getClient(); + const client = objects.getClient(); if (typeof key !== 'string') { throw new client.ErrorInfo('Map key should be string', 40003, 400); @@ -165,11 +162,11 @@ export class LiveMap extends LiveObject( - liveObjects: LiveObjects, + objects: Objects, key: TKey, value: API.LiveMapType[TKey], ): void { - const client = liveObjects.getClient(); + const client = objects.getClient(); if (typeof key !== 'string') { throw new client.ErrorInfo('Map key should be string', 40003, 400); @@ -189,14 +186,14 @@ export class LiveMap extends LiveObject { - const client = liveObjects.getClient(); + static async createMapCreateMessage(objects: Objects, entries?: API.LiveMapType): Promise { + const client = objects.getClient(); if (entries !== undefined && (entries === null || typeof entries !== 'object')) { throw new client.ErrorInfo('Map entries should be a key/value object', 40003, 400); } - Object.entries(entries ?? {}).forEach(([key, value]) => LiveMap.validateKeyValue(liveObjects, key, value)); + Object.entries(entries ?? {}).forEach(([key, value]) => LiveMap.validateKeyValue(objects, key, value)); const initialValueObj = LiveMap.createInitialValueObject(entries); const { encodedInitialValue, format } = StateMessage.encodeInitialValue(initialValueObj, client); @@ -266,7 +263,7 @@ export class LiveMap extends LiveObject(key: TKey): T[TKey] | undefined { - this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._objects.throwIfInvalidAccessApiConfiguration(); if (this.isTombstoned()) { return undefined as T[TKey]; @@ -287,7 +284,7 @@ export class LiveMap extends LiveObject extends LiveObject(): IterableIterator<[TKey, T[TKey]]> { - this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._objects.throwIfInvalidAccessApiConfiguration(); for (const [key, entry] of this._dataRef.data.entries()) { if (this._isMapEntryTombstoned(entry)) { @@ -339,9 +336,9 @@ export class LiveMap extends LiveObject(key: TKey, value: T[TKey]): Promise { - this._liveObjects.throwIfInvalidWriteApiConfiguration(); - const stateMessage = LiveMap.createMapSetMessage(this._liveObjects, this.getObjectId(), key, value); - return this._liveObjects.publish([stateMessage]); + this._objects.throwIfInvalidWriteApiConfiguration(); + const stateMessage = LiveMap.createMapSetMessage(this._objects, this.getObjectId(), key, value); + return this._objects.publish([stateMessage]); } /** @@ -354,9 +351,9 @@ export class LiveMap extends LiveObject(key: TKey): Promise { - this._liveObjects.throwIfInvalidWriteApiConfiguration(); - const stateMessage = LiveMap.createMapRemoveMessage(this._liveObjects, this.getObjectId(), key); - return this._liveObjects.publish([stateMessage]); + this._objects.throwIfInvalidWriteApiConfiguration(); + const stateMessage = LiveMap.createMapRemoveMessage(this._objects, this.getObjectId(), key); + return this._objects.publish([stateMessage]); } /** @@ -679,7 +676,7 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject void): SubscribeResponse { - this._liveObjects.throwIfInvalidAccessApiConfiguration(); + this._objects.throwIfInvalidAccessApiConfiguration(); this._subscriptions.on(LiveObjectSubscriptionEvent.updated, listener); @@ -235,7 +235,7 @@ export abstract class LiveObject< */ protected abstract _updateFromDataDiff(prevDataRef: TData, newDataRef: TData): TUpdate; /** - * Merges the initial data from the create operation into the live object state. + * Merges the initial data from the create operation into the live object. * * Client SDKs do not need to keep around the state operation that created the object, * so we can merge the initial data the first time we receive it for the object, diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/liveobjects/liveobjects.ts index 1ceed73aba..c1de3e7d8d 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/liveobjects/liveobjects.ts @@ -7,46 +7,46 @@ import { DEFAULTS } from './defaults'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; -import { LiveObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; +import { ObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; import { StateMessage, StateOperationAction } from './statemessage'; -import { SyncLiveObjectsDataPool } from './syncliveobjectsdatapool'; +import { SyncObjectsDataPool } from './syncliveobjectsdatapool'; -export enum LiveObjectsEvent { +export enum ObjectsEvent { syncing = 'syncing', synced = 'synced', } -export enum LiveObjectsState { +export enum ObjectsState { initialized = 'initialized', syncing = 'syncing', synced = 'synced', } -const StateToEventsMap: Record = { +const StateToEventsMap: Record = { initialized: undefined, - syncing: LiveObjectsEvent.syncing, - synced: LiveObjectsEvent.synced, + syncing: ObjectsEvent.syncing, + synced: ObjectsEvent.synced, }; -export type LiveObjectsEventCallback = () => void; +export type ObjectsEventCallback = () => void; -export interface OnLiveObjectsEventResponse { +export interface OnObjectsEventResponse { off(): void; } export type BatchCallback = (batchContext: BatchContext) => void; -export class LiveObjects { +export class Objects { private _client: BaseClient; private _channel: RealtimeChannel; - private _state: LiveObjectsState; + private _state: ObjectsState; // composition over inheritance since we cannot import class directly into plugin code. // instead we obtain a class type from the client private _eventEmitterInternal: EventEmitter; // related to RTC10, should have a separate EventEmitter for users of the library private _eventEmitterPublic: EventEmitter; - private _liveObjectsPool: LiveObjectsPool; - private _syncLiveObjectsDataPool: SyncLiveObjectsDataPool; + private _objectsPool: ObjectsPool; + private _syncObjectsDataPool: SyncObjectsDataPool; private _currentSyncId: string | undefined; private _currentSyncCursor: string | undefined; private _bufferedStateOperations: StateMessage[]; @@ -57,32 +57,32 @@ export class LiveObjects { constructor(channel: RealtimeChannel) { this._channel = channel; this._client = channel.client; - this._state = LiveObjectsState.initialized; + this._state = ObjectsState.initialized; this._eventEmitterInternal = new this._client.EventEmitter(this._client.logger); this._eventEmitterPublic = new this._client.EventEmitter(this._client.logger); - this._liveObjectsPool = new LiveObjectsPool(this); - this._syncLiveObjectsDataPool = new SyncLiveObjectsDataPool(this); + this._objectsPool = new ObjectsPool(this); + this._syncObjectsDataPool = new SyncObjectsDataPool(this); this._bufferedStateOperations = []; } /** - * When called without a type variable, we return a default root type which is based on globally defined LiveObjects interface. - * A user can provide an explicit type for the getRoot method to explicitly set the LiveObjects type structure on this particular channel. - * This is useful when working with LiveObjects on multiple channels with different underlying data. + * When called without a type variable, we return a default root type which is based on globally defined interface for Objects feature. + * A user can provide an explicit type for the getRoot method to explicitly set the type structure on this particular channel. + * This is useful when working with multiple channels with different underlying data structure. */ async getRoot(): Promise> { this.throwIfInvalidAccessApiConfiguration(); // if we're not synced yet, wait for SYNC sequence to finish before returning root - if (this._state !== LiveObjectsState.synced) { - await this._eventEmitterInternal.once(LiveObjectsEvent.synced); + if (this._state !== ObjectsState.synced) { + await this._eventEmitterInternal.once(ObjectsEvent.synced); } - return this._liveObjectsPool.get(ROOT_OBJECT_ID) as LiveMap; + return this._objectsPool.get(ROOT_OBJECT_ID) as LiveMap; } /** - * Provides access to the synchronous write API for LiveObjects that can be used to batch multiple operations together in a single channel message. + * Provides access to the synchronous write API for Objects that can be used to batch multiple operations together in a single channel message. */ async batch(callback: BatchCallback): Promise { this.throwIfInvalidWriteApiConfiguration(); @@ -117,15 +117,15 @@ export class LiveObjects { // we may have already received the CREATE operation at this point, as it could arrive before the ACK for our publish message. // this means the object might already exist in the local pool, having been added during the usual CREATE operation process. // here we check if the object is present, and return it if found; otherwise, create a new object on the client side. - if (this._liveObjectsPool.get(objectId)) { - return this._liveObjectsPool.get(objectId) as LiveMap; + if (this._objectsPool.get(objectId)) { + return this._objectsPool.get(objectId) as LiveMap; } // we haven't received the CREATE operation yet, so we can create a new map object using the locally constructed state operation. // we don't know the timeserials for map entries, so we assign an "earliest possible" timeserial to each entry, so that any subsequent operation can be applied to them. // we mark the CREATE operation as merged for the object, guaranteeing its idempotency and preventing it from being applied again when the operation arrives. const map = LiveMap.fromStateOperation(this, stateMessage.operation!); - this._liveObjectsPool.set(objectId, map); + this._objectsPool.set(objectId, map); return map; } @@ -149,19 +149,19 @@ export class LiveObjects { // we may have already received the CREATE operation at this point, as it could arrive before the ACK for our publish message. // this means the object might already exist in the local pool, having been added during the usual CREATE operation process. // here we check if the object is present, and return it if found; otherwise, create a new object on the client side. - if (this._liveObjectsPool.get(objectId)) { - return this._liveObjectsPool.get(objectId) as LiveCounter; + if (this._objectsPool.get(objectId)) { + return this._objectsPool.get(objectId) as LiveCounter; } // we haven't received the CREATE operation yet, so we can create a new counter object using the locally constructed state operation. // we mark the CREATE operation as merged for the object, guaranteeing its idempotency. this ensures we don't double count the initial counter value when the operation arrives. const counter = LiveCounter.fromStateOperation(this, stateMessage.operation!); - this._liveObjectsPool.set(objectId, counter); + this._objectsPool.set(objectId, counter); return counter; } - on(event: LiveObjectsEvent, callback: LiveObjectsEventCallback): OnLiveObjectsEventResponse { + on(event: ObjectsEvent, callback: ObjectsEventCallback): OnObjectsEventResponse { // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. this._eventEmitterPublic.on(event, callback); @@ -172,7 +172,7 @@ export class LiveObjects { return { off }; } - off(event: LiveObjectsEvent, callback: LiveObjectsEventCallback): void { + off(event: ObjectsEvent, callback: ObjectsEventCallback): void { // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. // prevent accidentally calling .off without any arguments on an EventEmitter and removing all callbacks @@ -191,8 +191,8 @@ export class LiveObjects { /** * @internal */ - getPool(): LiveObjectsPool { - return this._liveObjectsPool; + getPool(): ObjectsPool { + return this._objectsPool; } /** @@ -219,7 +219,7 @@ export class LiveObjects { this._startNewSync(syncId, syncCursor); } - this._syncLiveObjectsDataPool.applyStateSyncMessages(stateMessages); + this._syncObjectsDataPool.applyStateSyncMessages(stateMessages); // if this is the last (or only) message in a sequence of sync updates, end the sync if (!syncCursor) { @@ -233,7 +233,7 @@ export class LiveObjects { * @internal */ handleStateMessages(stateMessages: StateMessage[]): void { - if (this._state !== LiveObjectsState.synced) { + if (this._state !== ObjectsState.synced) { // The client receives state messages in realtime over the channel concurrently with the SYNC sequence. // Some of the incoming state messages may have already been applied to the state objects described in // the SYNC sequence, but others may not; therefore we must buffer these messages so that we can apply @@ -252,11 +252,11 @@ export class LiveObjects { this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MINOR, - 'LiveObjects.onAttached()', + 'Objects.onAttached()', `channel=${this._channel.name}, hasState=${hasState}`, ); - const fromInitializedState = this._state === LiveObjectsState.initialized; + const fromInitializedState = this._state === ObjectsState.initialized; if (hasState || fromInitializedState) { // should always start a new sync sequence if we're in the initialized state, no matter the HAS_STATE flag value. // this guarantees we emit both "syncing" -> "synced" events in that order. @@ -265,8 +265,8 @@ export class LiveObjects { if (!hasState) { // if no HAS_STATE flag received on attach, we can end SYNC sequence immediately and treat it as no state on a channel. - this._liveObjectsPool.reset(); - this._syncLiveObjectsDataPool.reset(); + this._objectsPool.reset(); + this._syncObjectsDataPool.reset(); // defer the state change event until the next tick if we started a new sequence just now due to being in initialized state. // this allows any event listeners to process the start of the new sequence event that was emitted earlier during this event loop. this._endSync(fromInitializedState); @@ -284,8 +284,8 @@ export class LiveObjects { case 'detached': case 'failed': - this._liveObjectsPool.reset(); - this._syncLiveObjectsDataPool.reset(); + this._objectsPool.reset(); + this._syncObjectsDataPool.reset(); break; } } @@ -320,7 +320,7 @@ export class LiveObjects { * @internal */ throwIfInvalidAccessApiConfiguration(): void { - this._throwIfMissingChannelMode('state_subscribe'); + this._throwIfMissingChannelMode('object_subscribe'); this._throwIfInChannelState(['detached', 'failed']); } @@ -328,17 +328,17 @@ export class LiveObjects { * @internal */ throwIfInvalidWriteApiConfiguration(): void { - this._throwIfMissingChannelMode('state_publish'); + this._throwIfMissingChannelMode('object_publish'); this._throwIfInChannelState(['detached', 'failed', 'suspended']); } private _startNewSync(syncId?: string, syncCursor?: string): void { // need to discard all buffered state operation messages on new sync start this._bufferedStateOperations = []; - this._syncLiveObjectsDataPool.reset(); + this._syncObjectsDataPool.reset(); this._currentSyncId = syncId; this._currentSyncCursor = syncCursor; - this._stateChange(LiveObjectsState.syncing, false); + this._stateChange(ObjectsState.syncing, false); } private _endSync(deferStateEvent: boolean): void { @@ -348,10 +348,10 @@ export class LiveObjects { this._applyStateMessages(this._bufferedStateOperations); this._bufferedStateOperations = []; - this._syncLiveObjectsDataPool.reset(); + this._syncObjectsDataPool.reset(); this._currentSyncId = undefined; this._currentSyncCursor = undefined; - this._stateChange(LiveObjectsState.synced, deferStateEvent); + this._stateChange(ObjectsState.synced, deferStateEvent); } private _parseSyncChannelSerial(syncChannelSerial: string | null | undefined): { @@ -373,16 +373,16 @@ export class LiveObjects { } private _applySync(): void { - if (this._syncLiveObjectsDataPool.isEmpty()) { + if (this._syncObjectsDataPool.isEmpty()) { return; } const receivedObjectIds = new Set(); const existingObjectUpdates: { object: LiveObject; update: LiveObjectUpdate | LiveObjectUpdateNoop }[] = []; - for (const [objectId, entry] of this._syncLiveObjectsDataPool.entries()) { + for (const [objectId, entry] of this._syncObjectsDataPool.entries()) { receivedObjectIds.add(objectId); - const existingObject = this._liveObjectsPool.get(objectId); + const existingObject = this._objectsPool.get(objectId); if (existingObject) { const update = existingObject.overrideWithStateObject(entry.stateObject); @@ -408,11 +408,11 @@ export class LiveObjects { throw new this._client.ErrorInfo(`Unknown Live Object type: ${objectType}`, 50000, 500); } - this._liveObjectsPool.set(objectId, newObject); + this._objectsPool.set(objectId, newObject); } - // need to remove LiveObject instances from the LiveObjectsPool for which objectIds were not received during the SYNC sequence - this._liveObjectsPool.deleteExtraObjectIds([...receivedObjectIds]); + // need to remove Live Object instances from the ObjectsPool for which objectIds were not received during the SYNC sequence + this._objectsPool.deleteExtraObjectIds([...receivedObjectIds]); // call subscription callbacks for all updated existing objects existingObjectUpdates.forEach(({ object, update }) => object.notifyUpdated(update)); @@ -424,7 +424,7 @@ export class LiveObjects { this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MAJOR, - 'LiveObjects._applyStateMessages()', + 'Objects._applyStateMessages()', `state operation message is received without 'operation' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, ); continue; @@ -445,22 +445,22 @@ export class LiveObjects { // since they need to be able to eventually initialize themselves from that *_CREATE op. // so to simplify operations handling, we always try to create a zero-value object in the pool first, // and then we can always apply the operation on the existing object in the pool. - this._liveObjectsPool.createZeroValueObjectIfNotExists(stateOperation.objectId); - this._liveObjectsPool.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); + this._objectsPool.createZeroValueObjectIfNotExists(stateOperation.objectId); + this._objectsPool.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); break; default: this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MAJOR, - 'LiveObjects._applyStateMessages()', + 'Objects._applyStateMessages()', `received unsupported action in state operation message: ${stateOperation.action}, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, ); } } } - private _throwIfMissingChannelMode(expectedMode: 'state_subscribe' | 'state_publish'): void { + private _throwIfMissingChannelMode(expectedMode: 'object_subscribe' | 'object_publish'): void { // channel.modes is only populated on channel attachment, so use it only if it is set, // otherwise as a best effort use user provided channel options if (this._channel.modes != null && !this._channel.modes.includes(expectedMode)) { @@ -471,7 +471,7 @@ export class LiveObjects { } } - private _stateChange(state: LiveObjectsState, deferEvent: boolean): void { + private _stateChange(state: ObjectsState, deferEvent: boolean): void { if (this._state === state) { return; } diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/liveobjects/liveobjectspool.ts index 428bdf867f..81485236e0 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/liveobjects/liveobjectspool.ts @@ -3,7 +3,7 @@ import { DEFAULTS } from './defaults'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; -import { LiveObjects } from './liveobjects'; +import { Objects } from './liveobjects'; import { ObjectId } from './objectid'; export const ROOT_OBJECT_ID = 'root'; @@ -11,13 +11,13 @@ export const ROOT_OBJECT_ID = 'root'; /** * @internal */ -export class LiveObjectsPool { +export class ObjectsPool { private _client: BaseClient; private _pool: Map; private _gcInterval: ReturnType; - constructor(private _liveObjects: LiveObjects) { - this._client = this._liveObjects.getClient(); + constructor(private _objects: Objects) { + this._client = this._objects.getClient(); this._pool = this._getInitialPool(); this._gcInterval = setInterval(() => { this._onGCInterval(); @@ -58,12 +58,12 @@ export class LiveObjectsPool { let zeroValueObject: LiveObject; switch (parsedObjectId.type) { case 'map': { - zeroValueObject = LiveMap.zeroValue(this._liveObjects, objectId); + zeroValueObject = LiveMap.zeroValue(this._objects, objectId); break; } case 'counter': - zeroValueObject = LiveCounter.zeroValue(this._liveObjects, objectId); + zeroValueObject = LiveCounter.zeroValue(this._objects, objectId); break; } @@ -73,7 +73,7 @@ export class LiveObjectsPool { private _getInitialPool(): Map { const pool = new Map(); - const root = LiveMap.zeroValue(this._liveObjects, ROOT_OBJECT_ID); + const root = LiveMap.zeroValue(this._objects, ROOT_OBJECT_ID); pool.set(root.getObjectId(), root); return pool; } @@ -82,7 +82,7 @@ export class LiveObjectsPool { const toDelete: string[] = []; for (const [objectId, obj] of this._pool.entries()) { // tombstoned objects should be removed from the pool if they have been tombstoned for longer than grace period. - // by removing them from the local pool, LiveObjects plugin no longer keeps a reference to those objects, allowing JS's + // by removing them from the local pool, Objects plugin no longer keeps a reference to those objects, allowing JS's // Garbage Collection to eventually free the memory for those objects, provided the user no longer references them either. if (obj.isTombstoned() && Date.now() - obj.tombstonedAt()! >= DEFAULTS.gcGracePeriod) { toDelete.push(objectId); diff --git a/src/plugins/liveobjects/syncliveobjectsdatapool.ts b/src/plugins/liveobjects/syncliveobjectsdatapool.ts index 663a1bb216..6c41e9045a 100644 --- a/src/plugins/liveobjects/syncliveobjectsdatapool.ts +++ b/src/plugins/liveobjects/syncliveobjectsdatapool.ts @@ -1,6 +1,6 @@ import type BaseClient from 'common/lib/client/baseclient'; import type RealtimeChannel from 'common/lib/client/realtimechannel'; -import { LiveObjects } from './liveobjects'; +import { Objects } from './liveobjects'; import { StateMessage, StateObject } from './statemessage'; export interface LiveObjectDataEntry { @@ -22,14 +22,14 @@ export type AnyDataEntry = LiveCounterDataEntry | LiveMapDataEntry; /** * @internal */ -export class SyncLiveObjectsDataPool { +export class SyncObjectsDataPool { private _client: BaseClient; private _channel: RealtimeChannel; private _pool: Map; - constructor(private _liveObjects: LiveObjects) { - this._client = this._liveObjects.getClient(); - this._channel = this._liveObjects.getChannel(); + constructor(private _objects: Objects) { + this._client = this._objects.getClient(); + this._channel = this._objects.getChannel(); this._pool = new Map(); } @@ -55,7 +55,7 @@ export class SyncLiveObjectsDataPool { this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MAJOR, - 'LiveObjects.SyncLiveObjectsDataPool.applyStateSyncMessages()', + 'SyncObjectsDataPool.applyStateSyncMessages()', `state object message is received during SYNC without 'object' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, ); continue; @@ -71,7 +71,7 @@ export class SyncLiveObjectsDataPool { this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MAJOR, - 'LiveObjects.SyncLiveObjectsDataPool.applyStateSyncMessages()', + 'SyncObjectsDataPool.applyStateSyncMessages()', `received unsupported state object message during SYNC, expected 'counter' or 'map' to be present, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, ); } diff --git a/test/common/globals/named_dependencies.js b/test/common/globals/named_dependencies.js index 1c26928249..88b4f0e992 100644 --- a/test/common/globals/named_dependencies.js +++ b/test/common/globals/named_dependencies.js @@ -11,9 +11,9 @@ define(function () { browser: 'build/push', node: 'build/push', }, - live_objects: { - browser: 'build/liveobjects', - node: 'build/liveobjects', + objects: { + browser: 'build/objects', + node: 'build/objects', }, // test modules @@ -27,7 +27,7 @@ define(function () { browser: 'test/common/modules/private_api_recorder', node: 'test/common/modules/private_api_recorder', }, - live_objects_helper: { + objects_helper: { browser: 'test/common/modules/live_objects_helper', node: 'test/common/modules/live_objects_helper', }, diff --git a/test/common/modules/live_objects_helper.js b/test/common/modules/live_objects_helper.js index aa35e77397..2580803103 100644 --- a/test/common/modules/live_objects_helper.js +++ b/test/common/modules/live_objects_helper.js @@ -1,10 +1,10 @@ 'use strict'; /** - * LiveObjects helper to create pre-determined state tree on channels + * Objects helper to create pre-determined state tree on channels */ -define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveObjectsPlugin) { - const createPM = Ably.makeProtocolMessageFromDeserialized({ LiveObjectsPlugin }); +define(['ably', 'shared_helper', 'objects'], function (Ably, Helper, ObjectsPlugin) { + const createPM = Ably.makeProtocolMessageFromDeserialized({ ObjectsPlugin }); const ACTIONS = { MAP_CREATE: 0, @@ -19,7 +19,7 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb return Helper.randomString(); } - class LiveObjectsHelper { + class ObjectsHelper { constructor(helper) { this._helper = helper; this._rest = helper.AblyRest({ useBinaryProtocol: false }); @@ -32,7 +32,7 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb } /** - * Sends REST STATE requests to create LiveObjects state tree on a provided channel name: + * Sends REST STATE requests to create Objects state tree on a provided channel name: * * root "emptyMap" -> Map#1 {} -- empty map * root "referencedMap" -> Map#2 { "counterKey": } @@ -311,7 +311,7 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb } const method = 'post'; - const path = `/channels/${channelName}/state`; + const path = `/channels/${channelName}/objects`; const response = await this._rest.request(method, path, 3, null, opBody, null); @@ -329,5 +329,5 @@ define(['ably', 'shared_helper', 'live_objects'], function (Ably, Helper, LiveOb } } - return (module.exports = LiveObjectsHelper); + return (module.exports = ObjectsHelper); }); diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index f1f07a2e17..e199afa6a2 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -18,8 +18,8 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'call.EventEmitter.emit', 'call.LiveObject.getObjectId', 'call.LiveObject.isTombstoned', - 'call.LiveObjects._liveObjectsPool._onGCInterval', - 'call.LiveObjects._liveObjectsPool.get', + 'call.Objects._objectsPool._onGCInterval', + 'call.Objects._objectsPool.get', 'call.Message.decode', 'call.Message.encode', 'call.Platform.Config.push.storage.clear', @@ -120,8 +120,8 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'read.transport.params.mode', 'read.transport.recvRequest.recvUri', 'read.transport.uri', - 'replace.LiveObjects._liveObjectsPool._onGCInterval', - 'replace.LiveObjects.publish', + 'replace.Objects._objectsPool._onGCInterval', + 'replace.Objects.publish', 'replace.channel.attachImpl', 'replace.channel.processMessage', 'replace.channel.sendMessage', @@ -138,8 +138,8 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'serialize.recoveryKey', 'write.Defaults.ENVIRONMENT', 'write.Defaults.wsConnectivityCheckUrl', - 'write.LiveObjects._DEFAULTS.gcGracePeriod', - 'write.LiveObjects._DEFAULTS.gcInterval', + 'write.Objects._DEFAULTS.gcGracePeriod', + 'write.Objects._DEFAULTS.gcInterval', 'write.Platform.Config.push', // This implies using a mock implementation of the internal IPlatformPushConfig interface. Our mock (in push_channel_transport.js) then interacts with internal objects and private APIs of public objects to implement this interface; I haven’t added annotations for that private API usage, since there wasn’t an easy way to pass test context information into the mock. I think that for now we can just say that if we wanted to get rid of this private API usage, then we’d need to remove this mock entirely. 'write.auth.authOptions.requestHeaders', 'write.auth.key', diff --git a/test/package/browser/template/README.md b/test/package/browser/template/README.md index 38f2e248d2..e42f76296c 100644 --- a/test/package/browser/template/README.md +++ b/test/package/browser/template/README.md @@ -8,7 +8,7 @@ This directory is intended to be used for testing the following aspects of the a It contains three files, each of which import ably-js in different manners, and provide a way to briefly exercise its functionality: - `src/index-default.ts` imports the default ably-js package (`import { Realtime } from 'ably'`). -- `src/index-liveobjects.ts` imports the LiveObjects ably-js plugin (`import LiveObjects from 'ably/liveobjects'`). +- `src/index-liveobjects.ts` imports the Objects ably-js plugin (`import Objects from 'ably/objects'`). - `src/index-modular.ts` imports the tree-shakable ably-js package (`import { BaseRealtime, WebSocketTransport, FetchRequest } from 'ably/modular'`). - `src/ReactApp.tsx` imports React hooks from the ably-js package (`import { useChannel } from 'ably/react'`). diff --git a/test/package/browser/template/server/resources/index-liveobjects.html b/test/package/browser/template/server/resources/index-liveobjects.html index b7f284af5a..be37a0bb26 100644 --- a/test/package/browser/template/server/resources/index-liveobjects.html +++ b/test/package/browser/template/server/resources/index-liveobjects.html @@ -2,7 +2,7 @@ - Ably NPM package test (LiveObjects plugin export) + Ably NPM package test (Objects plugin export) diff --git a/test/package/browser/template/src/ably.config.d.ts b/test/package/browser/template/src/ably.config.d.ts index 3b3c69ddb1..a9e596d3ab 100644 --- a/test/package/browser/template/src/ably.config.d.ts +++ b/test/package/browser/template/src/ably.config.d.ts @@ -15,7 +15,7 @@ type CustomRoot = { }; declare global { - export interface LiveObjectsTypes { + export interface ObjectsTypes { root: CustomRoot; } } diff --git a/test/package/browser/template/src/index-liveobjects.ts b/test/package/browser/template/src/index-liveobjects.ts index 4059cc01d8..269a9214a1 100644 --- a/test/package/browser/template/src/index-liveobjects.ts +++ b/test/package/browser/template/src/index-liveobjects.ts @@ -1,5 +1,5 @@ import * as Ably from 'ably'; -import LiveObjects from 'ably/liveobjects'; +import Objects from 'ably/objects'; import { CustomRoot } from './ably.config'; import { createSandboxAblyAPIKey } from './sandbox'; @@ -10,20 +10,20 @@ type ExplicitRootType = { globalThis.testAblyPackage = async function () { const key = await createSandboxAblyAPIKey({ featureFlags: ['enableChannelState'] }); - const realtime = new Ably.Realtime({ key, environment: 'sandbox', plugins: { LiveObjects } }); + const realtime = new Ably.Realtime({ key, environment: 'sandbox', plugins: { Objects } }); - const channel = realtime.channels.get('channel', { modes: ['STATE_SUBSCRIBE', 'STATE_PUBLISH'] }); - // check liveObjects can be accessed - const liveObjects = channel.liveObjects; + const channel = realtime.channels.get('channel', { modes: ['OBJECT_SUBSCRIBE', 'OBJECT_PUBLISH'] }); + // check Objects can be accessed + const objects = channel.objects; await channel.attach(); - // expect root to be a LiveMap instance with LiveObjects types defined via the global LiveObjectsTypes interface - // also checks that we can refer to the LiveObjects types exported from 'ably' by referencing a LiveMap interface - const root: Ably.LiveMap = await liveObjects.getRoot(); + // expect root to be a LiveMap instance with Objects types defined via the global ObjectsTypes interface + // also checks that we can refer to the Objects types exported from 'ably' by referencing a LiveMap interface + const root: Ably.LiveMap = await objects.getRoot(); // check root has expected LiveMap TypeScript type methods const size: number = root.size(); - // check custom user provided typings via LiveObjectsTypes are working: + // check custom user provided typings via ObjectsTypes are working: // any LiveMap.get() call can return undefined, as the LiveMap itself can be tombstoned (has empty state), // or referenced object is tombstoned. // keys on a root: @@ -33,7 +33,7 @@ globalThis.testAblyPackage = async function () { const userProvidedUndefined: string | undefined = root.get('couldBeUndefined'); // live objects on a root: const counter: Ably.LiveCounter | undefined = root.get('counterKey'); - const map: LiveObjectsTypes['root']['mapKey'] | undefined = root.get('mapKey'); + const map: ObjectsTypes['root']['mapKey'] | undefined = root.get('mapKey'); // check string literal types works // need to use nullish coalescing as we didn't actually create any data on the root, // so the next calls would fail. we only need to check that TypeScript types work @@ -61,7 +61,7 @@ globalThis.testAblyPackage = async function () { }); counterSubscribeResponse?.unsubscribe(); - // check can provide custom types for the getRoot method, ignoring global LiveObjectsTypes interface - const explicitRoot: Ably.LiveMap = await liveObjects.getRoot(); + // check can provide custom types for the getRoot method, ignoring global ObjectsTypes interface + const explicitRoot: Ably.LiveMap = await objects.getRoot(); const someOtherKey: string | undefined = explicitRoot.get('someOtherKey'); }; diff --git a/test/package/browser/template/test/lib/package.test.ts b/test/package/browser/template/test/lib/package.test.ts index 8554dd762b..159b1e0c34 100644 --- a/test/package/browser/template/test/lib/package.test.ts +++ b/test/package/browser/template/test/lib/package.test.ts @@ -3,7 +3,7 @@ import { test, expect } from '@playwright/test'; test.describe('NPM package', () => { for (const scenario of [ { name: 'default export', path: '/index-default.html' }, - { name: 'LiveObjects plugin export', path: '/index-liveobjects.html' }, + { name: 'Objects plugin export', path: '/index-liveobjects.html' }, { name: 'modular export', path: '/index-modular.html' }, ]) { test.describe(scenario.name, () => { diff --git a/test/realtime/live_objects.test.js b/test/realtime/live_objects.test.js index b98849ac76..83348b17e0 100644 --- a/test/realtime/live_objects.test.js +++ b/test/realtime/live_objects.test.js @@ -1,30 +1,30 @@ 'use strict'; -define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], function ( +define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ( Ably, Helper, chai, - LiveObjectsPlugin, - LiveObjectsHelper, + ObjectsPlugin, + ObjectsHelper, ) { const expect = chai.expect; const BufferUtils = Ably.Realtime.Platform.BufferUtils; const Utils = Ably.Realtime.Utils; const MessageEncoding = Ably.Realtime._MessageEncoding; - const createPM = Ably.makeProtocolMessageFromDeserialized({ LiveObjectsPlugin }); - const liveObjectsFixturesChannel = 'liveobjects_fixtures'; + const createPM = Ably.makeProtocolMessageFromDeserialized({ ObjectsPlugin }); + const objectsFixturesChannel = 'objects_fixtures'; const nextTick = Ably.Realtime.Platform.Config.nextTick; - const gcIntervalOriginal = LiveObjectsPlugin.LiveObjects._DEFAULTS.gcInterval; - const gcGracePeriodOriginal = LiveObjectsPlugin.LiveObjects._DEFAULTS.gcGracePeriod; + const gcIntervalOriginal = ObjectsPlugin.Objects._DEFAULTS.gcInterval; + const gcGracePeriodOriginal = ObjectsPlugin.Objects._DEFAULTS.gcGracePeriod; - function RealtimeWithLiveObjects(helper, options) { - return helper.AblyRealtime({ ...options, plugins: { LiveObjects: LiveObjectsPlugin } }); + function RealtimeWithObjects(helper, options) { + return helper.AblyRealtime({ ...options, plugins: { Objects: ObjectsPlugin } }); } - function channelOptionsWithLiveObjects(options) { + function channelOptionsWithObjects(options) { return { ...options, - modes: ['STATE_SUBSCRIBE', 'STATE_PUBLISH'], + modes: ['OBJECT_SUBSCRIBE', 'OBJECT_PUBLISH'], }; } @@ -88,7 +88,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], } function stateMessageFromValues(values) { - return LiveObjectsPlugin.StateMessage.fromValues(values, Utils, MessageEncoding); + return ObjectsPlugin.StateMessage.fromValues(values, Utils, MessageEncoding); } async function waitForMapKeyUpdate(map, key) { @@ -136,21 +136,21 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], } /** - * The channel with fixture data may not yet be populated by REST STATE requests made by LiveObjectsHelper. + * The channel with fixture data may not yet be populated by REST STATE requests made by ObjectsHelper. * This function waits for a channel to have all keys set. */ async function waitFixtureChannelIsReady(client) { - const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; - const expectedKeys = LiveObjectsHelper.fixtureRootKeys(); + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; + const expectedKeys = ObjectsHelper.fixtureRootKeys(); await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); await Promise.all(expectedKeys.map((key) => (root.get(key) ? undefined : waitForMapKeyUpdate(root, key)))); } - describe('realtime/live_objects', function () { + describe('realtime/objects', function () { this.timeout(60 * 1000); before(function (done) { @@ -162,26 +162,26 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], return; } - new LiveObjectsHelper(helper) - .initForChannel(liveObjectsFixturesChannel) + new ObjectsHelper(helper) + .initForChannel(objectsFixturesChannel) .then(done) .catch((err) => done(err)); }); }); - describe('Realtime without LiveObjects plugin', () => { + describe('Realtime without Objects plugin', () => { /** @nospec */ - it("throws an error when attempting to access the channel's `liveObjects` property", async function () { + it("throws an error when attempting to access the channel's `objects` property", async function () { const helper = this.test.helper; const client = helper.AblyRealtime({ autoConnect: false }); const channel = client.channels.get('channel'); - expect(() => channel.liveObjects).to.throw('LiveObjects plugin not provided'); + expect(() => channel.objects).to.throw('Objects plugin not provided'); }); /** @nospec */ it(`doesn't break when it receives a STATE ProtocolMessage`, async function () { const helper = this.test.helper; - const liveObjectsHelper = new LiveObjectsHelper(helper); + const objectsHelper = new ObjectsHelper(helper); const testClient = helper.AblyRealtime(); await helper.monitorConnectionThenCloseAndFinish(async () => { @@ -193,20 +193,18 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const publishClient = helper.AblyRealtime(); await helper.monitorConnectionThenCloseAndFinish(async () => { - // inject STATE message that should be ignored and not break anything without LiveObjects plugin - await liveObjectsHelper.processStateOperationMessageOnChannel({ + // inject STATE message that should be ignored and not break anything without the plugin + await objectsHelper.processStateOperationMessageOnChannel({ channel: testChannel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [ - liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'stringKey', data: { value: 'stringValue' } }), - ], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'stringKey', data: { value: 'stringValue' } })], }); const publishChannel = publishClient.channels.get('channel'); await publishChannel.publish(null, 'test'); - // regular message subscriptions should still work after processing STATE_SYNC message without LiveObjects plugin + // regular message subscriptions should still work after processing STATE_SYNC message without the plugin await receivedMessagePromise; }, publishClient); }, testClient); @@ -215,7 +213,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ it(`doesn't break when it receives a STATE_SYNC ProtocolMessage`, async function () { const helper = this.test.helper; - const liveObjectsHelper = new LiveObjectsHelper(helper); + const objectsHelper = new ObjectsHelper(helper); const testClient = helper.AblyRealtime(); await helper.monitorConnectionThenCloseAndFinish(async () => { @@ -227,12 +225,12 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const publishClient = helper.AblyRealtime(); await helper.monitorConnectionThenCloseAndFinish(async () => { - // inject STATE_SYNC message that should be ignored and not break anything without LiveObjects plugin - await liveObjectsHelper.processStateObjectMessageOnChannel({ + // inject STATE_SYNC message that should be ignored and not break anything without the plugin + await objectsHelper.processStateObjectMessageOnChannel({ channel: testChannel, syncSerial: 'serial:', state: [ - liveObjectsHelper.mapObject({ + objectsHelper.mapObject({ objectId: 'root', siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, }), @@ -242,33 +240,33 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const publishChannel = publishClient.channels.get('channel'); await publishChannel.publish(null, 'test'); - // regular message subscriptions should still work after processing STATE_SYNC message without LiveObjects plugin + // regular message subscriptions should still work after processing STATE_SYNC message without the plugin await receivedMessagePromise; }, publishClient); }, testClient); }); }); - describe('Realtime with LiveObjects plugin', () => { + describe('Realtime with Objects plugin', () => { /** @nospec */ - it("returns LiveObjects class instance when accessing channel's `liveObjects` property", async function () { + it("returns Objects class instance when accessing channel's `objects` property", async function () { const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper, { autoConnect: false }); + const client = RealtimeWithObjects(helper, { autoConnect: false }); const channel = client.channels.get('channel'); - expectInstanceOf(channel.liveObjects, 'LiveObjects'); + expectInstanceOf(channel.objects, 'Objects'); }); /** @nospec */ it('getRoot() returns LiveMap instance', async function () { const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper); + const client = RealtimeWithObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); expectInstanceOf(root, 'LiveMap', 'root object should be of LiveMap type'); }, client); @@ -277,14 +275,14 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ it('getRoot() returns live object with id "root"', async function () { const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper); + const client = RealtimeWithObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); helper.recordPrivateApi('call.LiveObject.getObjectId'); expect(root.getObjectId()).to.equal('root', 'root object should have an object id "root"'); @@ -294,14 +292,14 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ it('getRoot() returns empty root when no state exist on a channel', async function () { const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper); + const client = RealtimeWithObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); expect(root.size()).to.equal(0, 'Check root has no keys'); }, client); @@ -310,13 +308,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ it('getRoot() waits for initial STATE_SYNC to be completed before resolving', async function () { const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper); + const client = RealtimeWithObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; - const getRootPromise = liveObjects.getRoot(); + const getRootPromise = objects.getRoot(); let getRootResolved = false; getRootPromise.then(() => { @@ -338,18 +336,18 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ it('getRoot() resolves immediately when STATE_SYNC sequence is completed', async function () { const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper); + const client = RealtimeWithObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); // wait for STATE_SYNC sequence to complete by accessing root for the first time - await liveObjects.getRoot(); + await objects.getRoot(); let resolvedImmediately = false; - liveObjects.getRoot().then(() => { + objects.getRoot().then(() => { resolvedImmediately = true; }); @@ -364,19 +362,19 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ it('getRoot() waits for STATE_SYNC with empty cursor before resolving', async function () { const helper = this.test.helper; - const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper); + const objectsHelper = new ObjectsHelper(helper); + const client = RealtimeWithObjects(helper); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); // wait for initial STATE_SYNC sequence to complete - await liveObjects.getRoot(); + await objects.getRoot(); // inject STATE_SYNC message to emulate start of a new sequence - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, // have cursor so client awaits for additional STATE_SYNC messages syncSerial: 'serial:cursor', @@ -384,7 +382,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], let getRootResolved = false; let root; - liveObjects.getRoot().then((value) => { + objects.getRoot().then((value) => { getRootResolved = true; root = value; }); @@ -396,12 +394,12 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(getRootResolved, 'Check getRoot() is not resolved while STATE_SYNC is in progress').to.be.false; // inject final STATE_SYNC message - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, // no cursor to indicate the end of STATE_SYNC messages syncSerial: 'serial:', state: [ - liveObjectsHelper.mapObject({ + objectsHelper.mapObject({ objectId: 'root', siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, initialEntries: { key: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 1 } } }, @@ -425,16 +423,16 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], function (options, channelName) { return async function () { const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper, options); + const client = RealtimeWithObjects(helper, options); await helper.monitorConnectionThenCloseAndFinish(async () => { await waitFixtureChannelIsReady(client); - const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); const counterKeys = ['emptyCounter', 'initialValueCounter', 'referencedCounter']; const mapKeys = ['emptyMap', 'referencedMap', 'valuesMap']; @@ -488,16 +486,16 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], function (options, channelName) { return async function () { const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper, options); + const client = RealtimeWithObjects(helper, options); await helper.monitorConnectionThenCloseAndFinish(async () => { await waitFixtureChannelIsReady(client); - const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); const counters = [ { key: 'emptyCounter', value: 0 }, @@ -521,16 +519,16 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], function (options, channelName) { return async function () { const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper, options); + const client = RealtimeWithObjects(helper, options); await helper.monitorConnectionThenCloseAndFinish(async () => { await waitFixtureChannelIsReady(client); - const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); const emptyMap = root.get('emptyMap'); expect(emptyMap.size()).to.equal(0, 'Check empty map in root has no keys'); @@ -586,16 +584,16 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], function (options, channelName) { return async function () { const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper, options); + const client = RealtimeWithObjects(helper, options); await helper.monitorConnectionThenCloseAndFinish(async () => { await waitFixtureChannelIsReady(client); - const channel = client.channels.get(liveObjectsFixturesChannel, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); const referencedCounter = root.get('referencedCounter'); const referencedMap = root.get('referencedMap'); @@ -661,16 +659,16 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'STATE_SYNC sequence with state object "tombstone" property creates tombstoned object', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; - const mapId = liveObjectsHelper.fakeMapObjectId(); - const counterId = liveObjectsHelper.fakeCounterObjectId(); - await liveObjectsHelper.processStateObjectMessageOnChannel({ + const mapId = objectsHelper.fakeMapObjectId(); + const counterId = objectsHelper.fakeCounterObjectId(); + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately // add state objects with tombstone=true state: [ - liveObjectsHelper.mapObject({ + objectsHelper.mapObject({ objectId: mapId, siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0), @@ -678,7 +676,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], tombstone: true, initialEntries: {}, }), - liveObjectsHelper.counterObject({ + objectsHelper.counterObject({ objectId: counterId, siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0), @@ -686,7 +684,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], tombstone: true, initialCount: 1, }), - liveObjectsHelper.mapObject({ + objectsHelper.mapObject({ objectId: 'root', siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, initialEntries: { @@ -715,13 +713,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'STATE_SYNC sequence with state object "tombstone" property deletes existing object', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, channel } = ctx; + const { root, objectsHelper, channelName, channel } = ctx; const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); - const { objectId: counterId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: counterId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: liveObjectsHelper.counterCreateOp({ count: 1 }), + createOp: objectsHelper.counterCreateOp({ count: 1 }), }); await counterCreatedPromise; @@ -729,11 +727,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], .to.exist; // inject a STATE_SYNC sequence where a counter is now tombstoned - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately state: [ - liveObjectsHelper.counterObject({ + objectsHelper.counterObject({ objectId: counterId, siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0), @@ -741,7 +739,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], tombstone: true, initialCount: 1, }), - liveObjectsHelper.mapObject({ + objectsHelper.mapObject({ objectId: 'root', siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, initialEntries: { @@ -766,13 +764,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], description: 'STATE_SYNC sequence with state object "tombstone" property triggers subscription callback for existing object', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, channel } = ctx; + const { root, objectsHelper, channelName, channel } = ctx; const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); - const { objectId: counterId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: counterId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: liveObjectsHelper.counterCreateOp({ count: 1 }), + createOp: objectsHelper.counterCreateOp({ count: 1 }), }); await counterCreatedPromise; @@ -791,11 +789,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ); // inject a STATE_SYNC sequence where a counter is now tombstoned - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately state: [ - liveObjectsHelper.counterObject({ + objectsHelper.counterObject({ objectId: counterId, siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0), @@ -803,7 +801,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], tombstone: true, initialCount: 1, }), - liveObjectsHelper.mapObject({ + objectsHelper.mapObject({ objectId: 'root', siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, initialEntries: { @@ -823,9 +821,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can apply MAP_CREATE with primitives state operation messages', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, helper } = ctx; + const { root, objectsHelper, channelName, helper } = ctx; - // LiveObjects public API allows us to check value of objects we've created based on MAP_CREATE ops + // Objects public API allows us to check value of objects we've created based on MAP_CREATE ops // if we assign those objects to another map (root for example), as there is no way to access those objects from the internal pool directly. // however, in this test we put heavy focus on the data that is being created as the result of the MAP_CREATE op. @@ -840,10 +838,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // create new maps and set on root await Promise.all( primitiveMapsFixtures.map((fixture) => - liveObjectsHelper.createAndSetOnMap(channelName, { + objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: fixture.name, - createOp: liveObjectsHelper.mapCreateOp({ entries: fixture.entries }), + createOp: objectsHelper.mapCreateOp({ entries: fixture.entries }), }), ), ); @@ -887,10 +885,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can apply MAP_CREATE with object ids state operation messages', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; const withReferencesMapKey = 'withReferencesMap'; - // LiveObjects public API allows us to check value of objects we've created based on MAP_CREATE ops + // Objects public API allows us to check value of objects we've created based on MAP_CREATE ops // if we assign those objects to another map (root for example), as there is no way to access those objects from the internal pool directly. // however, in this test we put heavy focus on the data that is being created as the result of the MAP_CREATE op. @@ -902,18 +900,18 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const mapCreatedPromise = waitForMapKeyUpdate(root, withReferencesMapKey); // create map with references. need to create referenced objects first to obtain their object ids - const { objectId: referencedMapObjectId } = await liveObjectsHelper.stateRequest( + const { objectId: referencedMapObjectId } = await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapCreateOp({ entries: { stringKey: { data: { value: 'stringValue' } } } }), + objectsHelper.mapCreateOp({ entries: { stringKey: { data: { value: 'stringValue' } } } }), ); - const { objectId: referencedCounterObjectId } = await liveObjectsHelper.stateRequest( + const { objectId: referencedCounterObjectId } = await objectsHelper.stateRequest( channelName, - liveObjectsHelper.counterCreateOp({ count: 1 }), + objectsHelper.counterCreateOp({ count: 1 }), ); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: withReferencesMapKey, - createOp: liveObjectsHelper.mapCreateOp({ + createOp: objectsHelper.mapCreateOp({ entries: { mapReference: { data: { objectId: referencedMapObjectId } }, counterReference: { data: { objectId: referencedCounterObjectId } }, @@ -963,30 +961,30 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], description: 'MAP_CREATE state operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; // need to use multiple maps as MAP_CREATE op can only be applied once to a map object const mapIds = [ - liveObjectsHelper.fakeMapObjectId(), - liveObjectsHelper.fakeMapObjectId(), - liveObjectsHelper.fakeMapObjectId(), - liveObjectsHelper.fakeMapObjectId(), - liveObjectsHelper.fakeMapObjectId(), + objectsHelper.fakeMapObjectId(), + objectsHelper.fakeMapObjectId(), + objectsHelper.fakeMapObjectId(), + objectsHelper.fakeMapObjectId(), + objectsHelper.fakeMapObjectId(), ]; await Promise.all( mapIds.map(async (mapId, i) => { // send a MAP_SET op first to create a zero-value map with forged site timeserials vector (from the op), and set it on a root. - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', - state: [liveObjectsHelper.mapSetOp({ objectId: mapId, key: 'foo', data: { value: 'bar' } })], + state: [objectsHelper.mapSetOp({ objectId: mapId, key: 'foo', data: { value: 'bar' } })], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', - state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: mapId, data: { objectId: mapId } })], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: mapId, data: { objectId: mapId } })], }); }), ); @@ -999,12 +997,12 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied ].entries()) { - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial, siteCode, state: [ - liveObjectsHelper.mapCreateOp({ + objectsHelper.mapCreateOp({ objectId: mapIds[i], entries: { baz: { timeserial: serial, data: { value: 'qux' } }, @@ -1045,7 +1043,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can apply MAP_SET with primitives state operation messages', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, helper } = ctx; + const { root, objectsHelper, channelName, helper } = ctx; // check root is empty before ops primitiveKeyData.forEach((keyData) => { @@ -1059,9 +1057,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // apply MAP_SET ops await Promise.all( primitiveKeyData.map((keyData) => - liveObjectsHelper.stateRequest( + objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapSetOp({ + objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: keyData.data, @@ -1094,7 +1092,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can apply MAP_SET with object ids state operation messages', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; // check no object ids are set on root expect( @@ -1109,16 +1107,16 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], waitForMapKeyUpdate(root, 'keyToMap'), ]); // create new objects and set on root - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'keyToCounter', - createOp: liveObjectsHelper.counterCreateOp({ count: 1 }), + createOp: objectsHelper.counterCreateOp({ count: 1 }), }); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'keyToMap', - createOp: liveObjectsHelper.mapCreateOp({ + createOp: objectsHelper.mapCreateOp({ entries: { stringKey: { data: { value: 'stringValue' } }, }, @@ -1152,16 +1150,16 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], description: 'MAP_SET state operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; // create new map and set it on a root with forged timeserials - const mapId = liveObjectsHelper.fakeMapObjectId(); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + const mapId = objectsHelper.fakeMapObjectId(); + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', state: [ - liveObjectsHelper.mapCreateOp({ + objectsHelper.mapCreateOp({ objectId: mapId, entries: { foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, @@ -1174,11 +1172,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }), ], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'map', data: { objectId: mapId } })], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'map', data: { objectId: mapId } })], }); // inject operations with various timeserial values @@ -1190,11 +1188,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier entry CGO, not applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later entry CGO, applied ].entries()) { - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial, siteCode, - state: [liveObjectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { value: 'baz' } })], + state: [objectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { value: 'baz' } })], }); } @@ -1221,15 +1219,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can apply MAP_REMOVE state operation messages', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; const mapKey = 'map'; const mapCreatedPromise = waitForMapKeyUpdate(root, mapKey); // create new map and set on root - const { objectId: mapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: mapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: mapKey, - createOp: liveObjectsHelper.mapCreateOp({ + createOp: objectsHelper.mapCreateOp({ entries: { shouldStay: { data: { value: 'foo' } }, shouldDelete: { data: { value: 'bar' } }, @@ -1255,9 +1253,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const keyRemovedPromise = waitForMapKeyUpdate(map, 'shouldDelete'); // send MAP_REMOVE op - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapRemoveOp({ + objectsHelper.mapRemoveOp({ objectId: mapObjectId, key: 'shouldDelete', }), @@ -1284,16 +1282,16 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], description: 'MAP_REMOVE state operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; // create new map and set it on a root with forged timeserials - const mapId = liveObjectsHelper.fakeMapObjectId(); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + const mapId = objectsHelper.fakeMapObjectId(); + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', state: [ - liveObjectsHelper.mapCreateOp({ + objectsHelper.mapCreateOp({ objectId: mapId, entries: { foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, @@ -1306,11 +1304,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }), ], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'map', data: { objectId: mapId } })], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'map', data: { objectId: mapId } })], }); // inject operations with various timeserial values @@ -1322,11 +1320,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier entry CGO, not applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later entry CGO, applied ].entries()) { - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial, siteCode, - state: [liveObjectsHelper.mapRemoveOp({ objectId: mapId, key: `foo${i + 1}` })], + state: [objectsHelper.mapRemoveOp({ objectId: mapId, key: `foo${i + 1}` })], }); } @@ -1356,9 +1354,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can apply COUNTER_CREATE state operation messages', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; - // LiveObjects public API allows us to check value of objects we've created based on COUNTER_CREATE ops + // Objects public API allows us to check value of objects we've created based on COUNTER_CREATE ops // if we assign those objects to another map (root for example), as there is no way to access those objects from the internal pool directly. // however, in this test we put heavy focus on the data that is being created as the result of the COUNTER_CREATE op. @@ -1373,10 +1371,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // create new counters and set on root await Promise.all( countersFixtures.map((fixture) => - liveObjectsHelper.createAndSetOnMap(channelName, { + objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: fixture.name, - createOp: liveObjectsHelper.counterCreateOp({ count: fixture.count }), + createOp: objectsHelper.counterCreateOp({ count: fixture.count }), }), ), ); @@ -1409,32 +1407,30 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], description: 'COUNTER_CREATE state operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; // need to use multiple counters as COUNTER_CREATE op can only be applied once to a counter object const counterIds = [ - liveObjectsHelper.fakeCounterObjectId(), - liveObjectsHelper.fakeCounterObjectId(), - liveObjectsHelper.fakeCounterObjectId(), - liveObjectsHelper.fakeCounterObjectId(), - liveObjectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), ]; await Promise.all( counterIds.map(async (counterId, i) => { // send a COUNTER_INC op first to create a zero-value counter with forged site timeserials vector (from the op), and set it on a root. - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', - state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], + state: [objectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', - state: [ - liveObjectsHelper.mapSetOp({ objectId: 'root', key: counterId, data: { objectId: counterId } }), - ], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: counterId, data: { objectId: counterId } })], }); }), ); @@ -1447,11 +1443,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied ].entries()) { - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial, siteCode, - state: [liveObjectsHelper.counterCreateOp({ objectId: counterIds[i], count: 10 })], + state: [objectsHelper.counterCreateOp({ objectId: counterIds[i], count: 10 })], }); } @@ -1479,16 +1475,16 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can apply COUNTER_INC state operation messages', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; const counterKey = 'counter'; let expectedCounterValue = 0; const counterCreated = waitForMapKeyUpdate(root, counterKey); // create new counter and set on root - const { objectId: counterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: counterKey, - createOp: liveObjectsHelper.counterCreateOp({ count: expectedCounterValue }), + createOp: objectsHelper.counterCreateOp({ count: expectedCounterValue }), }); await counterCreated; @@ -1522,9 +1518,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expectedCounterValue += increment; const counterUpdatedPromise = waitForCounterUpdate(counter); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.counterIncOp({ + objectsHelper.counterIncOp({ objectId: counterObjectId, amount: increment, }), @@ -1543,21 +1539,21 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], description: 'COUNTER_INC state operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; // create new counter and set it on a root with forged timeserials - const counterId = liveObjectsHelper.fakeCounterObjectId(); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + const counterId = objectsHelper.fakeCounterObjectId(); + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', - state: [liveObjectsHelper.counterCreateOp({ objectId: counterId, count: 1 })], + state: [objectsHelper.counterCreateOp({ objectId: counterId, count: 1 })], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'counter', data: { objectId: counterId } })], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'counter', data: { objectId: counterId } })], }); // inject operations with various timeserial values @@ -1569,11 +1565,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // +100000 different site, earlier CGO, applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // +1000000 different site, later CGO, applied ].entries()) { - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial, siteCode, - state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: Math.pow(10, i + 1) })], + state: [objectsHelper.counterIncOp({ objectId: counterId, amount: Math.pow(10, i + 1) })], }); } @@ -1588,22 +1584,22 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'can apply OBJECT_DELETE state operation messages', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, channel } = ctx; + const { root, objectsHelper, channelName, channel } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'map'), waitForMapKeyUpdate(root, 'counter'), ]); // create initial objects and set on root - const { objectId: mapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: mapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: liveObjectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateOp(), }); - const { objectId: counterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: liveObjectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateOp(), }); await objectsCreatedPromise; @@ -1611,17 +1607,17 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(root.get('counter'), 'Check counter exists on root before OBJECT_DELETE').to.exist; // inject OBJECT_DELETE - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [liveObjectsHelper.objectDeleteOp({ objectId: mapObjectId })], + state: [objectsHelper.objectDeleteOp({ objectId: mapObjectId })], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 0), siteCode: 'aaa', - state: [liveObjectsHelper.objectDeleteOp({ objectId: counterObjectId })], + state: [objectsHelper.objectDeleteOp({ objectId: counterObjectId })], }); expect(root.get('map'), 'Check map is not accessible on root after OBJECT_DELETE').to.not.exist; @@ -1632,29 +1628,29 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'OBJECT_DELETE for unknown object id creates zero-value tombstoned object', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; - const counterId = liveObjectsHelper.fakeCounterObjectId(); + const counterId = objectsHelper.fakeCounterObjectId(); // inject OBJECT_DELETE. should create a zero-value tombstoned object which can't be modified - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [liveObjectsHelper.objectDeleteOp({ objectId: counterId })], + state: [objectsHelper.objectDeleteOp({ objectId: counterId })], }); // try to create and set tombstoned object on root - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb', - state: [liveObjectsHelper.counterCreateOp({ objectId: counterId })], + state: [objectsHelper.counterCreateOp({ objectId: counterId })], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', - state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'counter', data: { objectId: counterId } })], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'counter', data: { objectId: counterId } })], }); expect(root.get('counter'), 'Check counter is not accessible on root').to.not.exist; @@ -1665,32 +1661,30 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], description: 'OBJECT_DELETE state operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; // need to use multiple objects as OBJECT_DELETE op can only be applied once to an object const counterIds = [ - liveObjectsHelper.fakeCounterObjectId(), - liveObjectsHelper.fakeCounterObjectId(), - liveObjectsHelper.fakeCounterObjectId(), - liveObjectsHelper.fakeCounterObjectId(), - liveObjectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), ]; await Promise.all( counterIds.map(async (counterId, i) => { // create objects and set them on root with forged timeserials - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', - state: [liveObjectsHelper.counterCreateOp({ objectId: counterId })], + state: [objectsHelper.counterCreateOp({ objectId: counterId })], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', - state: [ - liveObjectsHelper.mapSetOp({ objectId: 'root', key: counterId, data: { objectId: counterId } }), - ], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: counterId, data: { objectId: counterId } })], }); }), ); @@ -1703,11 +1697,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied ].entries()) { - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial, siteCode, - state: [liveObjectsHelper.objectDeleteOp({ objectId: counterIds[i] })], + state: [objectsHelper.objectDeleteOp({ objectId: counterIds[i] })], }); } @@ -1741,27 +1735,27 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'OBJECT_DELETE triggers subscription callback with deleted data', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, channel } = ctx; + const { root, objectsHelper, channelName, channel } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'map'), waitForMapKeyUpdate(root, 'counter'), ]); // create initial objects and set on root - const { objectId: mapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: mapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: liveObjectsHelper.mapCreateOp({ + createOp: objectsHelper.mapCreateOp({ entries: { foo: { data: { value: 'bar' } }, baz: { data: { value: 1 } }, }, }), }); - const { objectId: counterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: liveObjectsHelper.counterCreateOp({ count: 1 }), + createOp: objectsHelper.counterCreateOp({ count: 1 }), }); await objectsCreatedPromise; @@ -1793,17 +1787,17 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ); // inject OBJECT_DELETE - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [liveObjectsHelper.objectDeleteOp({ objectId: mapObjectId })], + state: [objectsHelper.objectDeleteOp({ objectId: mapObjectId })], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 0), siteCode: 'aaa', - state: [liveObjectsHelper.objectDeleteOp({ objectId: counterObjectId })], + state: [objectsHelper.objectDeleteOp({ objectId: counterObjectId })], }); await Promise.all([mapSubPromise, counterSubPromise]); @@ -1813,35 +1807,33 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'MAP_SET with reference to a tombstoned object results in undefined value on key', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, channel } = ctx; + const { root, objectsHelper, channelName, channel } = ctx; const objectCreatedPromise = waitForMapKeyUpdate(root, 'foo'); // create initial objects and set on root - const { objectId: counterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'foo', - createOp: liveObjectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateOp(), }); await objectCreatedPromise; expect(root.get('foo'), 'Check counter exists on root before OBJECT_DELETE').to.exist; // inject OBJECT_DELETE - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [liveObjectsHelper.objectDeleteOp({ objectId: counterObjectId })], + state: [objectsHelper.objectDeleteOp({ objectId: counterObjectId })], }); // set tombstoned counter to another key on root - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [ - liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'bar', data: { objectId: counterObjectId } }), - ], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'bar', data: { objectId: counterObjectId } })], }); expect(root.get('bar'), 'Check counter is not accessible on new key in root after OBJECT_DELETE').to.not @@ -1852,7 +1844,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'state operation message on a tombstoned object does not revive it', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, channel } = ctx; + const { root, objectsHelper, channelName, channel } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'map1'), @@ -1860,20 +1852,20 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], waitForMapKeyUpdate(root, 'counter1'), ]); // create initial objects and set on root - const { objectId: mapId1 } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: mapId1 } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map1', - createOp: liveObjectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateOp(), }); - const { objectId: mapId2 } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: mapId2 } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map2', - createOp: liveObjectsHelper.mapCreateOp({ entries: { foo: { data: { value: 'bar' } } } }), + createOp: objectsHelper.mapCreateOp({ entries: { foo: { data: { value: 'bar' } } } }), }); - const { objectId: counterId1 } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: counterId1 } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter1', - createOp: liveObjectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateOp(), }); await objectsCreatedPromise; @@ -1882,43 +1874,43 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], expect(root.get('counter1'), 'Check counter1 exists on root before OBJECT_DELETE').to.exist; // inject OBJECT_DELETE - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [liveObjectsHelper.objectDeleteOp({ objectId: mapId1 })], + state: [objectsHelper.objectDeleteOp({ objectId: mapId1 })], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 0), siteCode: 'aaa', - state: [liveObjectsHelper.objectDeleteOp({ objectId: mapId2 })], + state: [objectsHelper.objectDeleteOp({ objectId: mapId2 })], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 2, 0), siteCode: 'aaa', - state: [liveObjectsHelper.objectDeleteOp({ objectId: counterId1 })], + state: [objectsHelper.objectDeleteOp({ objectId: counterId1 })], }); // inject state ops on tombstoned objects - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 3, 0), siteCode: 'aaa', - state: [liveObjectsHelper.mapSetOp({ objectId: mapId1, key: 'baz', data: { value: 'qux' } })], + state: [objectsHelper.mapSetOp({ objectId: mapId1, key: 'baz', data: { value: 'qux' } })], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 4, 0), siteCode: 'aaa', - state: [liveObjectsHelper.mapRemoveOp({ objectId: mapId2, key: 'foo' })], + state: [objectsHelper.mapRemoveOp({ objectId: mapId2, key: 'foo' })], }); - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 5, 0), siteCode: 'aaa', - state: [liveObjectsHelper.counterIncOp({ objectId: counterId1, amount: 1 })], + state: [objectsHelper.counterIncOp({ objectId: counterId1, amount: 1 })], }); // objects should still be deleted @@ -1938,10 +1930,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'state operation messages are buffered during STATE_SYNC sequence', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:cursor', }); @@ -1949,14 +1941,12 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // inject operations, it should not be applied as sync is in progress await Promise.all( primitiveKeyData.map((keyData) => - liveObjectsHelper.processStateOperationMessageOnChannel({ + objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', // copy data object as library will modify it - state: [ - liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } }), - ], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } })], }), ), ); @@ -1972,10 +1962,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'buffered state operation messages are applied when STATE_SYNC sequence ends', action: async (ctx) => { - const { root, liveObjectsHelper, channel, helper } = ctx; + const { root, objectsHelper, channel, helper } = ctx; // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:cursor', }); @@ -1983,20 +1973,18 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // inject operations, they should be applied when sync ends await Promise.all( primitiveKeyData.map((keyData, i) => - liveObjectsHelper.processStateOperationMessageOnChannel({ + objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', // copy data object as library will modify it - state: [ - liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } }), - ], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } })], }), ), ); // end the sync with empty cursor - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:', }); @@ -2023,10 +2011,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'buffered state operation messages are discarded when new STATE_SYNC sequence starts', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:cursor', }); @@ -2034,34 +2022,32 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // inject operations, expect them to be discarded when sync with new sequence id starts await Promise.all( primitiveKeyData.map((keyData, i) => - liveObjectsHelper.processStateOperationMessageOnChannel({ + objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', // copy data object as library will modify it - state: [ - liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } }), - ], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } })], }), ), ); // start new sync with new sequence id - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'otherserial:cursor', }); // inject another operation that should be applied when latest sync ends - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb', - state: [liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { value: 'bar' } })], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { value: 'bar' } })], }); // end sync - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'otherserial:', }); @@ -2086,18 +2072,18 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], description: 'buffered state operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages - const mapId = liveObjectsHelper.fakeMapObjectId(); - const counterId = liveObjectsHelper.fakeCounterObjectId(); - await liveObjectsHelper.processStateObjectMessageOnChannel({ + const mapId = objectsHelper.fakeMapObjectId(); + const counterId = objectsHelper.fakeCounterObjectId(); + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:cursor', // add state object messages with non-empty site timeserials state: [ // next map and counter objects will be checked to have correct operations applied on them based on site timeserials - liveObjectsHelper.mapObject({ + objectsHelper.mapObject({ objectId: mapId, siteTimeserials: { bbb: lexicoTimeserial('bbb', 2, 0), @@ -2114,7 +2100,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], foo8: { timeserial: lexicoTimeserial('ccc', 0, 0), data: { value: 'bar' } }, }, }), - liveObjectsHelper.counterObject({ + objectsHelper.counterObject({ objectId: counterId, siteTimeserials: { bbb: lexicoTimeserial('bbb', 1, 0), @@ -2122,7 +2108,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], initialCount: 1, }), // add objects to the root so they're discoverable in the state tree - liveObjectsHelper.mapObject({ + objectsHelper.mapObject({ objectId: 'root', siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, initialEntries: { @@ -2147,11 +2133,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // different site with matching entry CGO case is not possible, as matching entry timeserial means that that timeserial is in the site timeserials vector { serial: lexicoTimeserial('ddd', 1, 0), siteCode: 'ddd' }, // different site, later entry CGO, applied ].entries()) { - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial, siteCode, - state: [liveObjectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { value: 'baz' } })], + state: [objectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { value: 'baz' } })], }); } @@ -2164,16 +2150,16 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // +100000 different site, earlier CGO, applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // +1000000 different site, later CGO, applied ].entries()) { - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial, siteCode, - state: [liveObjectsHelper.counterIncOp({ objectId: counterId, amount: Math.pow(10, i + 1) })], + state: [objectsHelper.counterIncOp({ objectId: counterId, amount: Math.pow(10, i + 1) })], }); } // end sync - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:', }); @@ -2208,10 +2194,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], description: 'subsequent state operation messages are applied immediately after STATE_SYNC ended and buffers are applied', action: async (ctx) => { - const { root, liveObjectsHelper, channel, channelName, helper } = ctx; + const { root, objectsHelper, channel, channelName, helper } = ctx; // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:cursor', }); @@ -2219,29 +2205,27 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], // inject operations, they should be applied when sync ends await Promise.all( primitiveKeyData.map((keyData, i) => - liveObjectsHelper.processStateOperationMessageOnChannel({ + objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', // copy data object as library will modify it - state: [ - liveObjectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } }), - ], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } })], }), ), ); // end the sync with empty cursor - await liveObjectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:', }); const keyUpdatedPromise = waitForMapKeyUpdate(root, 'foo'); // send some more operations - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapSetOp({ + objectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { value: 'bar' }, @@ -2278,13 +2262,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'LiveCounter.increment sends COUNTER_INC operation', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: liveObjectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateOp(), }); await counterCreatedPromise; @@ -2321,13 +2305,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'LiveCounter.increment throws on invalid input', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: liveObjectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateOp(), }); await counterCreatedPromise; @@ -2388,13 +2372,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'LiveCounter.decrement sends COUNTER_INC operation', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: liveObjectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateOp(), }); await counterCreatedPromise; @@ -2431,13 +2415,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'LiveCounter.decrement throws on invalid input', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: liveObjectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateOp(), }); await counterCreatedPromise; @@ -2533,21 +2517,21 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'LiveMap.set sends MAP_SET operation with reference to another LiveObject', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'counter'), waitForMapKeyUpdate(root, 'map'), ]); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: liveObjectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateOp(), }); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: liveObjectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateOp(), }); await objectsCreatedPromise; @@ -2576,13 +2560,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'LiveMap.set throws on invalid input', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: liveObjectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateOp(), }); await mapCreatedPromise; @@ -2611,13 +2595,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'LiveMap.remove sends MAP_REMOVE operation', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: liveObjectsHelper.mapCreateOp({ + createOp: objectsHelper.mapCreateOp({ entries: { foo: { data: { value: 1 } }, bar: { data: { value: 1 } }, @@ -2646,13 +2630,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'LiveMap.remove throws on invalid input', action: async (ctx) => { - const { root, liveObjectsHelper, channelName } = ctx; + const { root, objectsHelper, channelName } = ctx; const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: liveObjectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateOp(), }); await mapCreatedPromise; @@ -2672,11 +2656,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { allTransportsAndProtocols: true, - description: 'LiveObjects.createCounter sends COUNTER_CREATE operation', + description: 'Objects.createCounter sends COUNTER_CREATE operation', action: async (ctx) => { - const { liveObjects } = ctx; + const { objects } = ctx; - const counters = await Promise.all(countersFixtures.map(async (x) => liveObjects.createCounter(x.count))); + const counters = await Promise.all(countersFixtures.map(async (x) => objects.createCounter(x.count))); for (let i = 0; i < counters.length; i++) { const counter = counters[i]; @@ -2694,12 +2678,12 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { allTransportsAndProtocols: true, - description: 'LiveCounter created with LiveObjects.createCounter can be assigned to the state tree', + description: 'LiveCounter created with Objects.createCounter can be assigned to the state tree', action: async (ctx) => { - const { root, liveObjects } = ctx; + const { root, objects } = ctx; const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); - const counter = await liveObjects.createCounter(1); + const counter = await objects.createCounter(1); await root.set('counter', counter); await counterCreatedPromise; @@ -2722,41 +2706,40 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: - 'LiveObjects.createCounter can return LiveCounter with initial value without applying CREATE operation', + 'Objects.createCounter can return LiveCounter with initial value without applying CREATE operation', action: async (ctx) => { - const { liveObjects, helper } = ctx; + const { objects, helper } = ctx; // prevent publishing of ops to realtime so we guarantee that the initial value doesn't come from a CREATE op - helper.recordPrivateApi('replace.LiveObjects.publish'); - liveObjects.publish = () => {}; + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = () => {}; - const counter = await liveObjects.createCounter(1); + const counter = await objects.createCounter(1); expect(counter.value()).to.equal(1, `Check counter has expected initial value`); }, }, { allTransportsAndProtocols: true, - description: - 'LiveObjects.createCounter can return LiveCounter with initial value from applied CREATE operation', + description: 'Objects.createCounter can return LiveCounter with initial value from applied CREATE operation', action: async (ctx) => { - const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + const { objects, objectsHelper, helper, channel } = ctx; // instead of sending CREATE op to the realtime, echo it immediately to the client // with forged initial value so we can check that counter gets initialized with a value from a CREATE op - helper.recordPrivateApi('replace.LiveObjects.publish'); - liveObjects.publish = async (stateMessages) => { + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = async (stateMessages) => { const counterId = stateMessages[0].operation.objectId; - // this should result in liveobjects' operation application procedure and create an object in the pool with forged initial value - await liveObjectsHelper.processStateOperationMessageOnChannel({ + // this should result execute regular operation application procedure and create an object in the pool with forged initial value + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 1), siteCode: 'aaa', - state: [liveObjectsHelper.counterCreateOp({ objectId: counterId, count: 10 })], + state: [objectsHelper.counterCreateOp({ objectId: counterId, count: 10 })], }); }; - const counter = await liveObjects.createCounter(1); + const counter = await objects.createCounter(1); // counter should be created with forged initial value instead of the actual one expect(counter.value()).to.equal( @@ -2768,25 +2751,25 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: - 'initial value is not double counted for LiveCounter from LiveObjects.createCounter when CREATE op is received', + 'initial value is not double counted for LiveCounter from Objects.createCounter when CREATE op is received', action: async (ctx) => { - const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + const { objects, objectsHelper, helper, channel } = ctx; // prevent publishing of ops to realtime so we can guarantee order of operations - helper.recordPrivateApi('replace.LiveObjects.publish'); - liveObjects.publish = () => {}; + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = () => {}; // create counter locally, should have an initial value set - const counter = await liveObjects.createCounter(1); + const counter = await objects.createCounter(1); helper.recordPrivateApi('call.LiveObject.getObjectId'); const counterId = counter.getObjectId(); // now inject CREATE op for a counter with a forged value. it should not be applied - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 1), siteCode: 'aaa', - state: [liveObjectsHelper.counterCreateOp({ objectId: counterId, count: 10 })], + state: [objectsHelper.counterCreateOp({ objectId: counterId, count: 10 })], }); expect(counter.value()).to.equal( @@ -2797,62 +2780,47 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { - description: 'LiveObjects.createCounter throws on invalid input', + description: 'Objects.createCounter throws on invalid input', action: async (ctx) => { - const { root, liveObjects } = ctx; + const { root, objects } = ctx; + await expectToThrowAsync(async () => objects.createCounter(null), 'Counter value should be a valid number'); await expectToThrowAsync( - async () => liveObjects.createCounter(null), - 'Counter value should be a valid number', - ); - await expectToThrowAsync( - async () => liveObjects.createCounter(Number.NaN), - 'Counter value should be a valid number', - ); - await expectToThrowAsync( - async () => liveObjects.createCounter(Number.POSITIVE_INFINITY), + async () => objects.createCounter(Number.NaN), 'Counter value should be a valid number', ); await expectToThrowAsync( - async () => liveObjects.createCounter(Number.NEGATIVE_INFINITY), + async () => objects.createCounter(Number.POSITIVE_INFINITY), 'Counter value should be a valid number', ); await expectToThrowAsync( - async () => liveObjects.createCounter('foo'), + async () => objects.createCounter(Number.NEGATIVE_INFINITY), 'Counter value should be a valid number', ); await expectToThrowAsync( - async () => liveObjects.createCounter(BigInt(1)), + async () => objects.createCounter('foo'), 'Counter value should be a valid number', ); await expectToThrowAsync( - async () => liveObjects.createCounter(true), + async () => objects.createCounter(BigInt(1)), 'Counter value should be a valid number', ); + await expectToThrowAsync(async () => objects.createCounter(true), 'Counter value should be a valid number'); await expectToThrowAsync( - async () => liveObjects.createCounter(Symbol()), - 'Counter value should be a valid number', - ); - await expectToThrowAsync( - async () => liveObjects.createCounter({}), - 'Counter value should be a valid number', - ); - await expectToThrowAsync( - async () => liveObjects.createCounter([]), - 'Counter value should be a valid number', - ); - await expectToThrowAsync( - async () => liveObjects.createCounter(root), + async () => objects.createCounter(Symbol()), 'Counter value should be a valid number', ); + await expectToThrowAsync(async () => objects.createCounter({}), 'Counter value should be a valid number'); + await expectToThrowAsync(async () => objects.createCounter([]), 'Counter value should be a valid number'); + await expectToThrowAsync(async () => objects.createCounter(root), 'Counter value should be a valid number'); }, }, { allTransportsAndProtocols: true, - description: 'LiveObjects.createMap sends MAP_CREATE operation with primitive values', + description: 'Objects.createMap sends MAP_CREATE operation with primitive values', action: async (ctx) => { - const { liveObjects, helper } = ctx; + const { objects, helper } = ctx; const maps = await Promise.all( primitiveMapsFixtures.map(async (mapFixture) => { @@ -2867,7 +2835,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, {}) : undefined; - return liveObjects.createMap(entries); + return objects.createMap(entries); }), ); @@ -2904,30 +2872,30 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { allTransportsAndProtocols: true, - description: 'LiveObjects.createMap sends MAP_CREATE operation with reference to another LiveObject', + description: 'Objects.createMap sends MAP_CREATE operation with reference to another LiveObject', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, liveObjects } = ctx; + const { root, objectsHelper, channelName, objects } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'counter'), waitForMapKeyUpdate(root, 'map'), ]); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: liveObjectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateOp(), }); - await liveObjectsHelper.createAndSetOnMap(channelName, { + await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: liveObjectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateOp(), }); await objectsCreatedPromise; const counter = root.get('counter'); const map = root.get('map'); - const newMap = await liveObjects.createMap({ counter, map }); + const newMap = await objects.createMap({ counter, map }); expect(newMap, 'Check map exists').to.exist; expectInstanceOf(newMap, 'LiveMap', 'Check map instance is of an expected class'); @@ -2945,13 +2913,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { allTransportsAndProtocols: true, - description: 'LiveMap created with LiveObjects.createMap can be assigned to the state tree', + description: 'LiveMap created with Objects.createMap can be assigned to the state tree', action: async (ctx) => { - const { root, liveObjects } = ctx; + const { root, objects } = ctx; const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); - const counter = await liveObjects.createCounter(); - const map = await liveObjects.createMap({ foo: 'bar', baz: counter }); + const counter = await objects.createCounter(); + const map = await objects.createMap({ foo: 'bar', baz: counter }); await root.set('map', map); await mapCreatedPromise; @@ -2974,37 +2942,37 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { - description: 'LiveObjects.createMap can return LiveMap with initial value without applying CREATE operation', + description: 'Objects.createMap can return LiveMap with initial value without applying CREATE operation', action: async (ctx) => { - const { liveObjects, helper } = ctx; + const { objects, helper } = ctx; // prevent publishing of ops to realtime so we guarantee that the initial value doesn't come from a CREATE op - helper.recordPrivateApi('replace.LiveObjects.publish'); - liveObjects.publish = () => {}; + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = () => {}; - const map = await liveObjects.createMap({ foo: 'bar' }); + const map = await objects.createMap({ foo: 'bar' }); expect(map.get('foo')).to.equal('bar', `Check map has expected initial value`); }, }, { allTransportsAndProtocols: true, - description: 'LiveObjects.createMap can return LiveMap with initial value from applied CREATE operation', + description: 'Objects.createMap can return LiveMap with initial value from applied CREATE operation', action: async (ctx) => { - const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + const { objects, objectsHelper, helper, channel } = ctx; // instead of sending CREATE op to the realtime, echo it immediately to the client // with forged initial value so we can check that map gets initialized with a value from a CREATE op - helper.recordPrivateApi('replace.LiveObjects.publish'); - liveObjects.publish = async (stateMessages) => { + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = async (stateMessages) => { const mapId = stateMessages[0].operation.objectId; - // this should result in liveobjects' operation application procedure and create an object in the pool with forged initial value - await liveObjectsHelper.processStateOperationMessageOnChannel({ + // this should result execute regular operation application procedure and create an object in the pool with forged initial value + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 1), siteCode: 'aaa', state: [ - liveObjectsHelper.mapCreateOp({ + objectsHelper.mapCreateOp({ objectId: mapId, entries: { baz: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { value: 'qux' } } }, }), @@ -3012,7 +2980,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); }; - const map = await liveObjects.createMap({ foo: 'bar' }); + const map = await objects.createMap({ foo: 'bar' }); // map should be created with forged initial value instead of the actual one expect(map.get('foo'), `Check key "foo" was not set on a map client-side`).to.not.exist; @@ -3025,26 +2993,26 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: - 'initial value is not double counted for LiveMap from LiveObjects.createMap when CREATE op is received', + 'initial value is not double counted for LiveMap from Objects.createMap when CREATE op is received', action: async (ctx) => { - const { liveObjects, liveObjectsHelper, helper, channel } = ctx; + const { objects, objectsHelper, helper, channel } = ctx; // prevent publishing of ops to realtime so we can guarantee order of operations - helper.recordPrivateApi('replace.LiveObjects.publish'); - liveObjects.publish = () => {}; + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = () => {}; // create map locally, should have an initial value set - const map = await liveObjects.createMap({ foo: 'bar' }); + const map = await objects.createMap({ foo: 'bar' }); helper.recordPrivateApi('call.LiveObject.getObjectId'); const mapId = map.getObjectId(); // now inject CREATE op for a map with a forged value. it should not be applied - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 1), siteCode: 'aaa', state: [ - liveObjectsHelper.mapCreateOp({ + objectsHelper.mapCreateOp({ objectId: mapId, entries: { foo: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { value: 'qux' } }, @@ -3064,65 +3032,50 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }, { - description: 'LiveObjects.createMap throws on invalid input', + description: 'Objects.createMap throws on invalid input', action: async (ctx) => { - const { root, liveObjects } = ctx; + const { root, objects } = ctx; + await expectToThrowAsync(async () => objects.createMap(null), 'Map entries should be a key/value object'); + await expectToThrowAsync(async () => objects.createMap('foo'), 'Map entries should be a key/value object'); + await expectToThrowAsync(async () => objects.createMap(1), 'Map entries should be a key/value object'); await expectToThrowAsync( - async () => liveObjects.createMap(null), + async () => objects.createMap(BigInt(1)), 'Map entries should be a key/value object', ); + await expectToThrowAsync(async () => objects.createMap(true), 'Map entries should be a key/value object'); await expectToThrowAsync( - async () => liveObjects.createMap('foo'), - 'Map entries should be a key/value object', - ); - await expectToThrowAsync(async () => liveObjects.createMap(1), 'Map entries should be a key/value object'); - await expectToThrowAsync( - async () => liveObjects.createMap(BigInt(1)), - 'Map entries should be a key/value object', - ); - await expectToThrowAsync( - async () => liveObjects.createMap(true), - 'Map entries should be a key/value object', - ); - await expectToThrowAsync( - async () => liveObjects.createMap(Symbol()), + async () => objects.createMap(Symbol()), 'Map entries should be a key/value object', ); await expectToThrowAsync( - async () => liveObjects.createMap({ key: undefined }), - 'Map value data type is unsupported', - ); - await expectToThrowAsync( - async () => liveObjects.createMap({ key: null }), + async () => objects.createMap({ key: undefined }), 'Map value data type is unsupported', ); await expectToThrowAsync( - async () => liveObjects.createMap({ key: BigInt(1) }), + async () => objects.createMap({ key: null }), 'Map value data type is unsupported', ); await expectToThrowAsync( - async () => liveObjects.createMap({ key: Symbol() }), + async () => objects.createMap({ key: BigInt(1) }), 'Map value data type is unsupported', ); await expectToThrowAsync( - async () => liveObjects.createMap({ key: {} }), - 'Map value data type is unsupported', - ); - await expectToThrowAsync( - async () => liveObjects.createMap({ key: [] }), + async () => objects.createMap({ key: Symbol() }), 'Map value data type is unsupported', ); + await expectToThrowAsync(async () => objects.createMap({ key: {} }), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => objects.createMap({ key: [] }), 'Map value data type is unsupported'); }, }, { description: 'batch API getRoot method is synchronous', action: async (ctx) => { - const { liveObjects } = ctx; + const { objects } = ctx; - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const root = ctx.getRoot(); expect(root, 'Check getRoot method in a BatchContext returns root object synchronously').to.exist; expectInstanceOf(root, 'LiveMap', 'root object obtained from a BatchContext is a LiveMap'); @@ -3133,19 +3086,19 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'batch API .get method on a map returns BatchContext* wrappers for live objects', action: async (ctx) => { - const { root, liveObjects } = ctx; + const { root, objects } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'counter'), waitForMapKeyUpdate(root, 'map'), ]); - const counter = await liveObjects.createCounter(1); - const map = await liveObjects.createMap({ innerCounter: counter }); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ innerCounter: counter }); await root.set('counter', counter); await root.set('map', map); await objectsCreatedPromise; - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const ctxRoot = ctx.getRoot(); const ctxCounter = ctxRoot.get('counter'); const ctxMap = ctxRoot.get('map'); @@ -3176,19 +3129,19 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'batch API access API methods on live objects work and are synchronous', action: async (ctx) => { - const { root, liveObjects } = ctx; + const { root, objects } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'counter'), waitForMapKeyUpdate(root, 'map'), ]); - const counter = await liveObjects.createCounter(1); - const map = await liveObjects.createMap({ foo: 'bar' }); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); await objectsCreatedPromise; - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const ctxRoot = ctx.getRoot(); const ctxCounter = ctxRoot.get('counter'); const ctxMap = ctxRoot.get('map'); @@ -3218,19 +3171,19 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'batch API write API methods on live objects do not mutate objects inside the batch callback', action: async (ctx) => { - const { root, liveObjects } = ctx; + const { root, objects } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'counter'), waitForMapKeyUpdate(root, 'map'), ]); - const counter = await liveObjects.createCounter(1); - const map = await liveObjects.createMap({ foo: 'bar' }); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); await objectsCreatedPromise; - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const ctxRoot = ctx.getRoot(); const ctxCounter = ctxRoot.get('counter'); const ctxMap = ctxRoot.get('map'); @@ -3266,19 +3219,19 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'batch API scheduled operations are applied when batch callback is finished', action: async (ctx) => { - const { root, liveObjects } = ctx; + const { root, objects } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'counter'), waitForMapKeyUpdate(root, 'map'), ]); - const counter = await liveObjects.createCounter(1); - const map = await liveObjects.createMap({ foo: 'bar' }); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); await objectsCreatedPromise; - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const ctxRoot = ctx.getRoot(); const ctxCounter = ctxRoot.get('counter'); const ctxMap = ctxRoot.get('map'); @@ -3299,11 +3252,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'batch API can be called without scheduling any operations', action: async (ctx) => { - const { liveObjects } = ctx; + const { objects } = ctx; let caughtError; try { - await liveObjects.batch((ctx) => {}); + await objects.batch((ctx) => {}); } catch (error) { caughtError = error; } @@ -3317,14 +3270,14 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'batch API scheduled operations can be canceled by throwing an error in the batch callback', action: async (ctx) => { - const { root, liveObjects } = ctx; + const { root, objects } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'counter'), waitForMapKeyUpdate(root, 'map'), ]); - const counter = await liveObjects.createCounter(1); - const map = await liveObjects.createMap({ foo: 'bar' }); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); await objectsCreatedPromise; @@ -3332,7 +3285,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const cancelError = new Error('cancel batch'); let caughtError; try { - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const ctxRoot = ctx.getRoot(); const ctxCounter = ctxRoot.get('counter'); const ctxMap = ctxRoot.get('map'); @@ -3362,14 +3315,14 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: `batch API batch context and derived objects can't be interacted with after the batch call`, action: async (ctx) => { - const { root, liveObjects } = ctx; + const { root, objects } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'counter'), waitForMapKeyUpdate(root, 'map'), ]); - const counter = await liveObjects.createCounter(1); - const map = await liveObjects.createMap({ foo: 'bar' }); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); await objectsCreatedPromise; @@ -3378,7 +3331,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], let savedCtxCounter; let savedCtxMap; - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const ctxRoot = ctx.getRoot(); savedCtx = ctx; savedCtxCounter = ctxRoot.get('counter'); @@ -3403,14 +3356,14 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: `batch API batch context and derived objects can't be interacted with after error was thrown from batch callback`, action: async (ctx) => { - const { root, liveObjects } = ctx; + const { root, objects } = ctx; const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'counter'), waitForMapKeyUpdate(root, 'map'), ]); - const counter = await liveObjects.createCounter(1); - const map = await liveObjects.createMap({ foo: 'bar' }); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); await root.set('counter', counter); await root.set('map', map); await objectsCreatedPromise; @@ -3421,7 +3374,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], let caughtError; try { - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const ctxRoot = ctx.getRoot(); savedCtx = ctx; savedCtxCounter = ctxRoot.get('counter'); @@ -3454,15 +3407,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: `LiveMap enumeration`, action: async (ctx) => { - const { root, liveObjectsHelper, channel } = ctx; + const { root, objectsHelper, channel } = ctx; - const counterId1 = liveObjectsHelper.fakeCounterObjectId(); - const counterId2 = liveObjectsHelper.fakeCounterObjectId(); - await liveObjectsHelper.processStateObjectMessageOnChannel({ + const counterId1 = objectsHelper.fakeCounterObjectId(); + const counterId2 = objectsHelper.fakeCounterObjectId(); + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately state: [ - liveObjectsHelper.counterObject({ + objectsHelper.counterObject({ objectId: counterId1, siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0), @@ -3470,7 +3423,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], tombstone: false, initialCount: 0, }), - liveObjectsHelper.counterObject({ + objectsHelper.counterObject({ objectId: counterId2, siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0), @@ -3478,7 +3431,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], tombstone: true, initialCount: 0, }), - liveObjectsHelper.mapObject({ + objectsHelper.mapObject({ objectId: 'root', siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, materialisedEntries: { @@ -3512,15 +3465,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: `BatchContextLiveMap enumeration`, action: async (ctx) => { - const { root, liveObjectsHelper, channel, liveObjects } = ctx; + const { root, objectsHelper, channel, objects } = ctx; - const counterId1 = liveObjectsHelper.fakeCounterObjectId(); - const counterId2 = liveObjectsHelper.fakeCounterObjectId(); - await liveObjectsHelper.processStateObjectMessageOnChannel({ + const counterId1 = objectsHelper.fakeCounterObjectId(); + const counterId2 = objectsHelper.fakeCounterObjectId(); + await objectsHelper.processStateObjectMessageOnChannel({ channel, syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately state: [ - liveObjectsHelper.counterObject({ + objectsHelper.counterObject({ objectId: counterId1, siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0), @@ -3528,7 +3481,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], tombstone: false, initialCount: 0, }), - liveObjectsHelper.counterObject({ + objectsHelper.counterObject({ objectId: counterId2, siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0), @@ -3536,7 +3489,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], tombstone: true, initialCount: 0, }), - liveObjectsHelper.mapObject({ + objectsHelper.mapObject({ objectId: 'root', siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, materialisedEntries: { @@ -3551,7 +3504,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const counter1 = await root.get('counter1'); - await liveObjects.batch(async (ctx) => { + await objects.batch(async (ctx) => { const ctxRoot = ctx.getRoot(); // enumeration methods should not count tombstoned entries @@ -3587,17 +3540,17 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ...liveMapEnumerationScenarios, ], async function (helper, scenario, clientOptions, channelName) { - const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper, clientOptions); + const objectsHelper = new ObjectsHelper(helper); + const client = RealtimeWithObjects(helper, clientOptions); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get(channelName, channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); - await scenario.action({ liveObjects, root, liveObjectsHelper, channelName, channel, client, helper }); + await scenario.action({ objects, root, objectsHelper, channelName, channel, client, helper }); }, client); }, ); @@ -3607,7 +3560,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can subscribe to the incoming COUNTER_INC operation on a LiveCounter', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + const { root, objectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; const counter = root.get(sampleCounterKey); const subscriptionPromise = new Promise((resolve, reject) => @@ -3624,9 +3577,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }), ); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.counterIncOp({ + objectsHelper.counterIncOp({ objectId: sampleCounterObjectId, amount: 1, }), @@ -3640,7 +3593,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can subscribe to multiple incoming operations on a LiveCounter', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + const { root, objectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; const counter = root.get(sampleCounterKey); const expectedCounterIncrements = [100, -100, Number.MAX_SAFE_INTEGER, -Number.MAX_SAFE_INTEGER]; @@ -3667,9 +3620,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], ); for (const increment of expectedCounterIncrements) { - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.counterIncOp({ + objectsHelper.counterIncOp({ objectId: sampleCounterObjectId, amount: increment, }), @@ -3684,7 +3637,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can subscribe to the incoming MAP_SET operation on a LiveMap', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; const map = root.get(sampleMapKey); const subscriptionPromise = new Promise((resolve, reject) => @@ -3701,9 +3654,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }), ); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapSetOp({ + objectsHelper.mapSetOp({ objectId: sampleMapObjectId, key: 'stringKey', data: { value: 'stringValue' }, @@ -3718,7 +3671,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can subscribe to the incoming MAP_REMOVE operation on a LiveMap', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; const map = root.get(sampleMapKey); const subscriptionPromise = new Promise((resolve, reject) => @@ -3735,9 +3688,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }), ); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapRemoveOp({ + objectsHelper.mapRemoveOp({ objectId: sampleMapObjectId, key: 'stringKey', }), @@ -3751,7 +3704,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'can subscribe to multiple incoming operations on a LiveMap', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; const map = root.get(sampleMapKey); const expectedMapUpdates = [ @@ -3782,44 +3735,44 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }), ); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapSetOp({ + objectsHelper.mapSetOp({ objectId: sampleMapObjectId, key: 'foo', data: { value: 'something' }, }), ); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapSetOp({ + objectsHelper.mapSetOp({ objectId: sampleMapObjectId, key: 'bar', data: { value: 'something' }, }), ); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapRemoveOp({ + objectsHelper.mapRemoveOp({ objectId: sampleMapObjectId, key: 'foo', }), ); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapSetOp({ + objectsHelper.mapSetOp({ objectId: sampleMapObjectId, key: 'baz', data: { value: 'something' }, }), ); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapRemoveOp({ + objectsHelper.mapRemoveOp({ objectId: sampleMapObjectId, key: 'bar', }), @@ -3832,7 +3785,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'can unsubscribe from LiveCounter updates via returned "unsubscribe" callback', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + const { root, objectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; const counter = root.get(sampleCounterKey); let callbackCalled = 0; @@ -3848,9 +3801,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const increments = 3; for (let i = 0; i < increments; i++) { const counterUpdatedPromise = waitForCounterUpdate(counter); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.counterIncOp({ + objectsHelper.counterIncOp({ objectId: sampleCounterObjectId, amount: 1, }), @@ -3868,7 +3821,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'can unsubscribe from LiveCounter updates via LiveCounter.unsubscribe() call', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + const { root, objectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; const counter = root.get(sampleCounterKey); let callbackCalled = 0; @@ -3886,9 +3839,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const increments = 3; for (let i = 0; i < increments; i++) { const counterUpdatedPromise = waitForCounterUpdate(counter); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.counterIncOp({ + objectsHelper.counterIncOp({ objectId: sampleCounterObjectId, amount: 1, }), @@ -3906,7 +3859,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'can remove all LiveCounter update listeners via LiveCounter.unsubscribeAll() call', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + const { root, objectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; const counter = root.get(sampleCounterKey); const callbacks = 3; @@ -3926,9 +3879,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const increments = 3; for (let i = 0; i < increments; i++) { const counterUpdatedPromise = waitForCounterUpdate(counter); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.counterIncOp({ + objectsHelper.counterIncOp({ objectId: sampleCounterObjectId, amount: 1, }), @@ -3951,7 +3904,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'can unsubscribe from LiveMap updates via returned "unsubscribe" callback', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; const map = root.get(sampleMapKey); let callbackCalled = 0; @@ -3967,9 +3920,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const mapSets = 3; for (let i = 0; i < mapSets; i++) { const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapSetOp({ + objectsHelper.mapSetOp({ objectId: sampleMapObjectId, key: `foo-${i}`, data: { value: 'exists' }, @@ -3993,7 +3946,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'can unsubscribe from LiveMap updates via LiveMap.unsubscribe() call', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; const map = root.get(sampleMapKey); let callbackCalled = 0; @@ -4011,9 +3964,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const mapSets = 3; for (let i = 0; i < mapSets; i++) { const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapSetOp({ + objectsHelper.mapSetOp({ objectId: sampleMapObjectId, key: `foo-${i}`, data: { value: 'exists' }, @@ -4037,7 +3990,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'can remove all LiveMap update listeners via LiveMap.unsubscribeAll() call', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; const map = root.get(sampleMapKey); const callbacks = 3; @@ -4057,9 +4010,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const mapSets = 3; for (let i = 0; i < mapSets; i++) { const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapSetOp({ + objectsHelper.mapSetOp({ objectId: sampleMapObjectId, key: `foo-${i}`, data: { value: 'exists' }, @@ -4088,15 +4041,15 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ forScenarios(this, subscriptionCallbacksScenarios, async function (helper, scenario, clientOptions, channelName) { - const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper, clientOptions); + const objectsHelper = new ObjectsHelper(helper); + const client = RealtimeWithObjects(helper, clientOptions); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get(channelName, channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); const sampleMapKey = 'sampleMap'; const sampleCounterKey = 'sampleCounter'; @@ -4106,21 +4059,21 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], waitForMapKeyUpdate(root, sampleCounterKey), ]); // prepare map and counter objects for use by the scenario - const { objectId: sampleMapObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: sampleMapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: sampleMapKey, - createOp: liveObjectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateOp(), }); - const { objectId: sampleCounterObjectId } = await liveObjectsHelper.createAndSetOnMap(channelName, { + const { objectId: sampleCounterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: sampleCounterKey, - createOp: liveObjectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateOp(), }); await objectsCreatedPromise; await scenario.action({ root, - liveObjectsHelper, + objectsHelper, channelName, channel, sampleMapKey, @@ -4132,45 +4085,40 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); const tombstonesGCScenarios = [ - // for the next tests we need to access the private API of LiveObjects plugin in order to verify that tombstoned entities were indeed deleted after the GC grace period. + // for the next tests we need to access the private API of Objects plugin in order to verify that tombstoned entities were indeed deleted after the GC grace period. // public API hides that kind of information from the user and returns undefined for tombstoned entities even if realtime client still keeps a reference to them. { description: 'tombstoned object is removed from the pool after the GC grace period', action: async (ctx) => { - const { liveObjectsHelper, channelName, channel, liveObjects, helper, waitForGCCycles, client } = ctx; + const { objectsHelper, channelName, channel, objects, helper, waitForGCCycles, client } = ctx; - const counterCreatedPromise = waitForStateOperation( - helper, - client, - LiveObjectsHelper.ACTIONS.COUNTER_CREATE, - ); + const counterCreatedPromise = waitForStateOperation(helper, client, ObjectsHelper.ACTIONS.COUNTER_CREATE); // send a CREATE op, this adds an object to the pool - const { objectId } = await liveObjectsHelper.stateRequest( + const { objectId } = await objectsHelper.stateRequest( channelName, - liveObjectsHelper.counterCreateOp({ count: 1 }), + objectsHelper.counterCreateOp({ count: 1 }), ); await counterCreatedPromise; - helper.recordPrivateApi('call.LiveObjects._liveObjectsPool.get'); - expect(liveObjects._liveObjectsPool.get(objectId), 'Check object exists in the pool after creation').to - .exist; + helper.recordPrivateApi('call.Objects._objectsPool.get'); + expect(objects._objectsPool.get(objectId), 'Check object exists in the pool after creation').to.exist; // inject OBJECT_DELETE for the object. this should tombstone the object and make it inaccessible to the end user, but still keep it in memory in the local pool - await liveObjectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processStateOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [liveObjectsHelper.objectDeleteOp({ objectId })], + state: [objectsHelper.objectDeleteOp({ objectId })], }); - helper.recordPrivateApi('call.LiveObjects._liveObjectsPool.get'); + helper.recordPrivateApi('call.Objects._objectsPool.get'); expect( - liveObjects._liveObjectsPool.get(objectId), + objects._objectsPool.get(objectId), 'Check object exists in the pool immediately after OBJECT_DELETE', ).to.exist; - helper.recordPrivateApi('call.LiveObjects._liveObjectsPool.get'); + helper.recordPrivateApi('call.Objects._objectsPool.get'); helper.recordPrivateApi('call.LiveObject.isTombstoned'); - expect(liveObjects._liveObjectsPool.get(objectId).isTombstoned()).to.equal( + expect(objects._objectsPool.get(objectId).isTombstoned()).to.equal( true, `Check object's "tombstone" flag is set to "true" after OBJECT_DELETE`, ); @@ -4179,9 +4127,9 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await waitForGCCycles(2); // object should be removed from the local pool entirely now, as the GC grace period has passed - helper.recordPrivateApi('call.LiveObjects._liveObjectsPool.get'); + helper.recordPrivateApi('call.Objects._objectsPool.get'); expect( - liveObjects._liveObjectsPool.get(objectId), + objects._objectsPool.get(objectId), 'Check object exists does not exist in the pool after the GC grace period expiration', ).to.not.exist; }, @@ -4191,13 +4139,13 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], allTransportsAndProtocols: true, description: 'tombstoned map entry is removed from the LiveMap after the GC grace period', action: async (ctx) => { - const { root, liveObjectsHelper, channelName, helper, waitForGCCycles } = ctx; + const { root, objectsHelper, channelName, helper, waitForGCCycles } = ctx; const keyUpdatedPromise = waitForMapKeyUpdate(root, 'foo'); // set a key on a root - await liveObjectsHelper.stateRequest( + await objectsHelper.stateRequest( channelName, - liveObjectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { value: 'bar' } }), + objectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { value: 'bar' } }), ); await keyUpdatedPromise; @@ -4205,10 +4153,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], const keyUpdatedPromise2 = waitForMapKeyUpdate(root, 'foo'); // remove the key from the root. this should tombstone the map entry and make it inaccessible to the end user, but still keep it in memory in the underlying map - await liveObjectsHelper.stateRequest( - channelName, - liveObjectsHelper.mapRemoveOp({ objectId: 'root', key: 'foo' }), - ); + await objectsHelper.stateRequest(channelName, objectsHelper.mapRemoveOp({ objectId: 'root', key: 'foo' })); await keyUpdatedPromise2; expect(root.get('foo'), 'Check key "foo" is inaccessible via public API on root after MAP_REMOVE').to.not @@ -4240,36 +4185,36 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ forScenarios(this, tombstonesGCScenarios, async function (helper, scenario, clientOptions, channelName) { try { - helper.recordPrivateApi('write.LiveObjects._DEFAULTS.gcInterval'); - LiveObjectsPlugin.LiveObjects._DEFAULTS.gcInterval = 500; - helper.recordPrivateApi('write.LiveObjects._DEFAULTS.gcGracePeriod'); - LiveObjectsPlugin.LiveObjects._DEFAULTS.gcGracePeriod = 250; + helper.recordPrivateApi('write.Objects._DEFAULTS.gcInterval'); + ObjectsPlugin.Objects._DEFAULTS.gcInterval = 500; + helper.recordPrivateApi('write.Objects._DEFAULTS.gcGracePeriod'); + ObjectsPlugin.Objects._DEFAULTS.gcGracePeriod = 250; - const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper, clientOptions); + const objectsHelper = new ObjectsHelper(helper); + const client = RealtimeWithObjects(helper, clientOptions); await helper.monitorConnectionThenCloseAndFinish(async () => { - const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get(channelName, channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); // helper function to spy on the GC interval callback and wait for a specific number of GC cycles. // returns a promise which will resolve when required number of cycles have happened. const waitForGCCycles = (cycles) => { - const onGCIntervalOriginal = liveObjects._liveObjectsPool._onGCInterval; + const onGCIntervalOriginal = objects._objectsPool._onGCInterval; let gcCalledTimes = 0; return new Promise((resolve) => { - helper.recordPrivateApi('replace.LiveObjects._liveObjectsPool._onGCInterval'); - liveObjects._liveObjectsPool._onGCInterval = function () { - helper.recordPrivateApi('call.LiveObjects._liveObjectsPool._onGCInterval'); + helper.recordPrivateApi('replace.Objects._objectsPool._onGCInterval'); + objects._objectsPool._onGCInterval = function () { + helper.recordPrivateApi('call.Objects._objectsPool._onGCInterval'); onGCIntervalOriginal.call(this); gcCalledTimes++; if (gcCalledTimes >= cycles) { resolve(); - liveObjects._liveObjectsPool._onGCInterval = onGCIntervalOriginal; + objects._objectsPool._onGCInterval = onGCIntervalOriginal; } }; }); @@ -4278,24 +4223,24 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], await scenario.action({ client, root, - liveObjectsHelper, + objectsHelper, channelName, channel, - liveObjects, + objects, helper, waitForGCCycles, }); }, client); } finally { - helper.recordPrivateApi('write.LiveObjects._DEFAULTS.gcInterval'); - LiveObjectsPlugin.LiveObjects._DEFAULTS.gcInterval = gcIntervalOriginal; - helper.recordPrivateApi('write.LiveObjects._DEFAULTS.gcGracePeriod'); - LiveObjectsPlugin.LiveObjects._DEFAULTS.gcGracePeriod = gcGracePeriodOriginal; + helper.recordPrivateApi('write.Objects._DEFAULTS.gcInterval'); + ObjectsPlugin.Objects._DEFAULTS.gcInterval = gcIntervalOriginal; + helper.recordPrivateApi('write.Objects._DEFAULTS.gcGracePeriod'); + ObjectsPlugin.Objects._DEFAULTS.gcGracePeriod = gcGracePeriodOriginal; } }); - const expectAccessApiToThrow = async ({ liveObjects, map, counter, errorMsg }) => { - await expectToThrowAsync(async () => liveObjects.getRoot(), errorMsg); + const expectAccessApiToThrow = async ({ objects, map, counter, errorMsg }) => { + await expectToThrowAsync(async () => objects.getRoot(), errorMsg); expect(() => counter.value()).to.throw(errorMsg); @@ -4312,10 +4257,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], } }; - const expectWriteApiToThrow = async ({ liveObjects, map, counter, errorMsg }) => { - await expectToThrowAsync(async () => liveObjects.batch(), errorMsg); - await expectToThrowAsync(async () => liveObjects.createMap(), errorMsg); - await expectToThrowAsync(async () => liveObjects.createCounter(), errorMsg); + const expectWriteApiToThrow = async ({ objects, map, counter, errorMsg }) => { + await expectToThrowAsync(async () => objects.batch(), errorMsg); + await expectToThrowAsync(async () => objects.createMap(), errorMsg); + await expectToThrowAsync(async () => objects.createCounter(), errorMsg); await expectToThrowAsync(async () => counter.increment(), errorMsg); await expectToThrowAsync(async () => counter.decrement(), errorMsg); @@ -4355,21 +4300,21 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], { description: 'public API throws missing state modes error when attached without correct state modes', action: async (ctx) => { - const { liveObjects, channel, map, counter } = ctx; + const { objects, channel, map, counter } = ctx; // obtain batch context with valid modes first - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const map = ctx.getRoot().get('map'); const counter = ctx.getRoot().get('counter'); // now simulate missing modes channel.modes = []; - expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: '"state_subscribe" channel mode' }); - expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: '"state_publish" channel mode' }); + expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: '"object_subscribe" channel mode' }); + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: '"object_publish" channel mode' }); }); - await expectAccessApiToThrow({ liveObjects, map, counter, errorMsg: '"state_subscribe" channel mode' }); - await expectWriteApiToThrow({ liveObjects, map, counter, errorMsg: '"state_publish" channel mode' }); + await expectAccessApiToThrow({ objects, map, counter, errorMsg: '"object_subscribe" channel mode' }); + await expectWriteApiToThrow({ objects, map, counter, errorMsg: '"object_publish" channel mode' }); }, }, @@ -4377,10 +4322,10 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], description: 'public API throws missing state modes error when not yet attached but client options are missing correct modes', action: async (ctx) => { - const { liveObjects, channel, map, counter, helper } = ctx; + const { objects, channel, map, counter, helper } = ctx; // obtain batch context with valid modes first - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const map = ctx.getRoot().get('map'); const counter = ctx.getRoot().get('counter'); // now simulate a situation where we're not yet attached/modes are not received on ATTACHED event @@ -4388,22 +4333,22 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], helper.recordPrivateApi('write.channel.channelOptions.modes'); channel.channelOptions.modes = []; - expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: '"state_subscribe" channel mode' }); - expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: '"state_publish" channel mode' }); + expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: '"object_subscribe" channel mode' }); + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: '"object_publish" channel mode' }); }); - await expectAccessApiToThrow({ liveObjects, map, counter, errorMsg: '"state_subscribe" channel mode' }); - await expectWriteApiToThrow({ liveObjects, map, counter, errorMsg: '"state_publish" channel mode' }); + await expectAccessApiToThrow({ objects, map, counter, errorMsg: '"object_subscribe" channel mode' }); + await expectWriteApiToThrow({ objects, map, counter, errorMsg: '"object_publish" channel mode' }); }, }, { description: 'public API throws invalid channel state error when channel DETACHED', action: async (ctx) => { - const { liveObjects, channel, map, counter, helper } = ctx; + const { objects, channel, map, counter, helper } = ctx; // obtain batch context with valid channel state first - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const map = ctx.getRoot().get('map'); const counter = ctx.getRoot().get('counter'); // now simulate channel state change @@ -4415,22 +4360,22 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); await expectAccessApiToThrow({ - liveObjects, + objects, map, counter, errorMsg: 'failed as channel state is detached', }); - await expectWriteApiToThrow({ liveObjects, map, counter, errorMsg: 'failed as channel state is detached' }); + await expectWriteApiToThrow({ objects, map, counter, errorMsg: 'failed as channel state is detached' }); }, }, { description: 'public API throws invalid channel state error when channel FAILED', action: async (ctx) => { - const { liveObjects, channel, map, counter, helper } = ctx; + const { objects, channel, map, counter, helper } = ctx; // obtain batch context with valid channel state first - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const map = ctx.getRoot().get('map'); const counter = ctx.getRoot().get('counter'); // now simulate channel state change @@ -4442,22 +4387,22 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); await expectAccessApiToThrow({ - liveObjects, + objects, map, counter, errorMsg: 'failed as channel state is failed', }); - await expectWriteApiToThrow({ liveObjects, map, counter, errorMsg: 'failed as channel state is failed' }); + await expectWriteApiToThrow({ objects, map, counter, errorMsg: 'failed as channel state is failed' }); }, }, { description: 'public write API throws invalid channel state error when channel SUSPENDED', action: async (ctx) => { - const { liveObjects, channel, map, counter, helper } = ctx; + const { objects, channel, map, counter, helper } = ctx; // obtain batch context with valid channel state first - await liveObjects.batch((ctx) => { + await objects.batch((ctx) => { const map = ctx.getRoot().get('map'); const counter = ctx.getRoot().get('counter'); // now simulate channel state change @@ -4468,7 +4413,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); await expectWriteApiToThrow({ - liveObjects, + objects, map, counter, errorMsg: 'failed as channel state is suspended', @@ -4479,29 +4424,29 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ forScenarios(this, channelConfigurationScenarios, async function (helper, scenario, clientOptions, channelName) { - const liveObjectsHelper = new LiveObjectsHelper(helper); - const client = RealtimeWithLiveObjects(helper, clientOptions); + const objectsHelper = new ObjectsHelper(helper); + const client = RealtimeWithObjects(helper, clientOptions); await helper.monitorConnectionThenCloseAndFinish(async () => { - // attach with correct channel modes so we can create liveobjects on the root for testing. + // attach with correct channel modes so we can create Objects on the root for testing. // some scenarios will modify the underlying modes array to test specific behavior - const channel = client.channels.get(channelName, channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get(channelName, channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); const objectsCreatedPromise = Promise.all([ waitForMapKeyUpdate(root, 'map'), waitForMapKeyUpdate(root, 'counter'), ]); - const map = await liveObjects.createMap(); - const counter = await liveObjects.createCounter(); + const map = await objects.createMap(); + const counter = await objects.createCounter(); await root.set('map', map); await root.set('counter', counter); await objectsCreatedPromise; - await scenario.action({ liveObjects, liveObjectsHelper, channelName, channel, root, map, counter, helper }); + await scenario.action({ objects, objectsHelper, channelName, channel, root, map, counter, helper }); }, client); }); @@ -4511,7 +4456,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], */ it('state message publish respects connectionDetails.maxMessageSize', async function () { const helper = this.test.helper; - const client = RealtimeWithLiveObjects(helper, { clientId: 'test' }); + const client = RealtimeWithObjects(helper, { clientId: 'test' }); await helper.monitorConnectionThenCloseAndFinish(async () => { await client.connection.once('connected'); @@ -4537,11 +4482,11 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], helper.recordPrivateApi('listen.connectionManager.connectiondetails'); await connectionDetailsPromise; - const channel = client.channels.get('channel', channelOptionsWithLiveObjects()); - const liveObjects = channel.liveObjects; + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; await channel.attach(); - const root = await liveObjects.getRoot(); + const root = await objects.getRoot(); const data = new Array(100).fill('a').join(''); const error = await expectToThrowAsync( @@ -4767,7 +4712,7 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], /** @nospec */ forScenarios(this, stateMessageSizeScenarios, function (helper, scenario) { helper.recordPrivateApi('call.StateMessage.encode'); - LiveObjectsPlugin.StateMessage.encode(scenario.message); + ObjectsPlugin.StateMessage.encode(scenario.message); helper.recordPrivateApi('call.BufferUtils.utf8Encode'); // was called by a scenario to create buffers helper.recordPrivateApi('call.StateMessage.fromValues'); // was called by a scenario to create a StateMessage instance helper.recordPrivateApi('call.Utils.dataSizeBytes'); // was called by a scenario to calculated the expected byte size @@ -4778,20 +4723,20 @@ define(['ably', 'shared_helper', 'chai', 'live_objects', 'live_objects_helper'], }); /** @nospec */ - it('can attach to channel with LiveObjects state modes', async function () { + it('can attach to channel with Objects state modes', async function () { const helper = this.test.helper; const client = helper.AblyRealtime(); await helper.monitorConnectionThenCloseAndFinish(async () => { - const liveObjectsModes = ['state_subscribe', 'state_publish']; - const channelOptions = { modes: liveObjectsModes }; + const objectsModes = ['object_subscribe', 'object_publish']; + const channelOptions = { modes: objectsModes }; const channel = client.channels.get('channel', channelOptions); await channel.attach(); helper.recordPrivateApi('read.channel.channelOptions'); expect(channel.channelOptions).to.deep.equal(channelOptions, 'Check expected channel options'); - expect(channel.modes).to.deep.equal(liveObjectsModes, 'Check expected modes'); + expect(channel.modes).to.deep.equal(objectsModes, 'Check expected modes'); }, client); }); }); diff --git a/typedoc.json b/typedoc.json index 7c9c174766..4474605cda 100644 --- a/typedoc.json +++ b/typedoc.json @@ -21,5 +21,5 @@ "Variable", "Namespace" ], - "intentionallyNotExported": ["__global.LiveObjectsTypes"] + "intentionallyNotExported": ["__global.ObjectsTypes"] } From 706d4ba64082c66af84e1ec3bf3f552e77902139 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 5 Mar 2025 04:25:12 +0000 Subject: [PATCH 136/166] Rename "LiveObjects" file names to just "Objects" --- grunt/esbuild/build.js | 6 ++--- liveobjects.d.ts => objects.d.ts | 0 package.json | 4 ++-- scripts/moduleReport.ts | 24 +++++++++---------- src/common/lib/client/baserealtime.ts | 2 +- src/common/lib/client/modularplugins.ts | 2 +- src/common/lib/client/realtimechannel.ts | 2 +- src/common/lib/types/protocolmessage.ts | 2 +- src/plugins/index.d.ts | 2 +- .../{liveobjects => objects}/batchcontext.ts | 4 ++-- .../batchcontextlivecounter.ts | 2 +- .../batchcontextlivemap.ts | 2 +- .../{liveobjects => objects}/defaults.ts | 0 src/plugins/{liveobjects => objects}/index.ts | 2 +- .../{liveobjects => objects}/livecounter.ts | 2 +- .../{liveobjects => objects}/livemap.ts | 2 +- .../{liveobjects => objects}/liveobject.ts | 2 +- .../{liveobjects => objects}/objectid.ts | 0 .../liveobjects.ts => objects/objects.ts} | 4 ++-- .../objectspool.ts} | 2 +- .../{liveobjects => objects}/statemessage.ts | 0 .../syncobjectsdatapool.ts} | 2 +- test/common/globals/named_dependencies.js | 4 ++-- ...ve_objects_helper.js => objects_helper.js} | 0 test/package/browser/template/README.md | 4 ++-- test/package/browser/template/package.json | 2 +- ...ex-liveobjects.html => index-objects.html} | 2 +- .../package/browser/template/server/server.ts | 2 +- ...{index-liveobjects.ts => index-objects.ts} | 0 .../browser/template/test/lib/package.test.ts | 2 +- .../{live_objects.test.js => objects.test.js} | 0 test/support/browser_file_list.js | 2 +- 32 files changed, 43 insertions(+), 43 deletions(-) rename liveobjects.d.ts => objects.d.ts (100%) rename src/plugins/{liveobjects => objects}/batchcontext.ts (96%) rename src/plugins/{liveobjects => objects}/batchcontextlivecounter.ts (96%) rename src/plugins/{liveobjects => objects}/batchcontextlivemap.ts (98%) rename src/plugins/{liveobjects => objects}/defaults.ts (100%) rename src/plugins/{liveobjects => objects}/index.ts (76%) rename src/plugins/{liveobjects => objects}/livecounter.ts (99%) rename src/plugins/{liveobjects => objects}/livemap.ts (99%) rename src/plugins/{liveobjects => objects}/liveobject.ts (99%) rename src/plugins/{liveobjects => objects}/objectid.ts (100%) rename src/plugins/{liveobjects/liveobjects.ts => objects/objects.ts} (99%) rename src/plugins/{liveobjects/liveobjectspool.ts => objects/objectspool.ts} (98%) rename src/plugins/{liveobjects => objects}/statemessage.ts (100%) rename src/plugins/{liveobjects/syncliveobjectsdatapool.ts => objects/syncobjectsdatapool.ts} (98%) rename test/common/modules/{live_objects_helper.js => objects_helper.js} (100%) rename test/package/browser/template/server/resources/{index-liveobjects.html => index-objects.html} (76%) rename test/package/browser/template/src/{index-liveobjects.ts => index-objects.ts} (100%) rename test/realtime/{live_objects.test.js => objects.test.js} (100%) diff --git a/grunt/esbuild/build.js b/grunt/esbuild/build.js index 772a9e07ea..ec809f4bde 100644 --- a/grunt/esbuild/build.js +++ b/grunt/esbuild/build.js @@ -79,7 +79,7 @@ const minifiedPushPluginCdnConfig = { const objectsPluginConfig = { ...createBaseConfig(), - entryPoints: ['src/plugins/liveobjects/index.ts'], + entryPoints: ['src/plugins/objects/index.ts'], plugins: [umdWrapper.default({ libraryName: 'AblyObjectsPlugin', amdNamedModule: false })], outfile: 'build/objects.js', external: ['deep-equal'], @@ -87,14 +87,14 @@ const objectsPluginConfig = { const objectsPluginCdnConfig = { ...createBaseConfig(), - entryPoints: ['src/plugins/liveobjects/index.ts'], + entryPoints: ['src/plugins/objects/index.ts'], plugins: [umdWrapper.default({ libraryName: 'AblyObjectsPlugin', amdNamedModule: false })], outfile: 'build/objects.umd.js', }; const minifiedObjectsPluginCdnConfig = { ...createBaseConfig(), - entryPoints: ['src/plugins/liveobjects/index.ts'], + entryPoints: ['src/plugins/objects/index.ts'], plugins: [umdWrapper.default({ libraryName: 'AblyObjectsPlugin', amdNamedModule: false })], outfile: 'build/objects.umd.min.js', minify: true, diff --git a/liveobjects.d.ts b/objects.d.ts similarity index 100% rename from liveobjects.d.ts rename to objects.d.ts diff --git a/package.json b/package.json index 69921c95e7..74e63ee8ba 100644 --- a/package.json +++ b/package.json @@ -31,14 +31,14 @@ "import": "./build/push.js" }, "./objects": { - "types": "./liveobjects.d.ts", + "types": "./objects.d.ts", "import": "./build/objects.js" } }, "files": [ "build/**", "ably.d.ts", - "liveobjects.d.ts", + "objects.d.ts", "modular.d.ts", "push.d.ts", "resources/**", diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index a6d80645e8..a79a6abd75 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -323,18 +323,18 @@ async function checkObjectsPluginFiles() { // These are the files that are allowed to contribute >= `threshold` bytes to the Objects bundle. const allowedFiles = new Set([ - 'src/plugins/liveobjects/batchcontext.ts', - 'src/plugins/liveobjects/batchcontextlivecounter.ts', - 'src/plugins/liveobjects/batchcontextlivemap.ts', - 'src/plugins/liveobjects/index.ts', - 'src/plugins/liveobjects/livecounter.ts', - 'src/plugins/liveobjects/livemap.ts', - 'src/plugins/liveobjects/liveobject.ts', - 'src/plugins/liveobjects/liveobjects.ts', - 'src/plugins/liveobjects/liveobjectspool.ts', - 'src/plugins/liveobjects/objectid.ts', - 'src/plugins/liveobjects/statemessage.ts', - 'src/plugins/liveobjects/syncliveobjectsdatapool.ts', + 'src/plugins/objects/batchcontext.ts', + 'src/plugins/objects/batchcontextlivecounter.ts', + 'src/plugins/objects/batchcontextlivemap.ts', + 'src/plugins/objects/index.ts', + 'src/plugins/objects/livecounter.ts', + 'src/plugins/objects/livemap.ts', + 'src/plugins/objects/liveobject.ts', + 'src/plugins/objects/objectid.ts', + 'src/plugins/objects/objects.ts', + 'src/plugins/objects/objectspool.ts', + 'src/plugins/objects/statemessage.ts', + 'src/plugins/objects/syncobjectsdatapool.ts', ]); return checkBundleFiles(pluginBundleInfo, allowedFiles, 100); diff --git a/src/common/lib/client/baserealtime.ts b/src/common/lib/client/baserealtime.ts index ee02b1a01e..af3bdc78b1 100644 --- a/src/common/lib/client/baserealtime.ts +++ b/src/common/lib/client/baserealtime.ts @@ -13,7 +13,7 @@ import { ModularPlugins, RealtimePresencePlugin } from './modularplugins'; import { TransportNames } from 'common/constants/TransportName'; import { TransportImplementations } from 'common/platform'; import Defaults from '../util/defaults'; -import type * as ObjectsPlugin from 'plugins/liveobjects'; +import type * as ObjectsPlugin from 'plugins/objects'; /** `BaseRealtime` is an export of the tree-shakable version of the SDK, and acts as the base class for the `DefaultRealtime` class exported by the non tree-shakable version. diff --git a/src/common/lib/client/modularplugins.ts b/src/common/lib/client/modularplugins.ts index 244d867db9..adad4cd92b 100644 --- a/src/common/lib/client/modularplugins.ts +++ b/src/common/lib/client/modularplugins.ts @@ -11,7 +11,7 @@ import { } from '../types/presencemessage'; import { TransportCtor } from '../transport/transport'; import type * as PushPlugin from 'plugins/push'; -import type * as ObjectsPlugin from 'plugins/liveobjects'; +import type * as ObjectsPlugin from 'plugins/objects'; export interface PresenceMessagePlugin { presenceMessageFromValues: typeof presenceMessageFromValues; diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index 0a22ad8686..6dff30af9c 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -30,7 +30,7 @@ import { ChannelOptions } from '../../types/channel'; import { normaliseChannelOptions } from '../util/defaults'; import { PaginatedResult } from './paginatedresource'; import type { PushChannel } from 'plugins/push'; -import type { Objects, StateMessage } from 'plugins/liveobjects'; +import type { Objects, StateMessage } from 'plugins/objects'; interface RealtimeHistoryParams { start?: number; diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index 1750e14346..455f25105c 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -12,7 +12,7 @@ import PresenceMessage, { fromValues as presenceMessageFromValues, fromValuesArray as presenceMessagesFromValuesArray, } from './presencemessage'; -import type * as ObjectsPlugin from 'plugins/liveobjects'; +import type * as ObjectsPlugin from 'plugins/objects'; export const actions = { HEARTBEAT: 0, diff --git a/src/plugins/index.d.ts b/src/plugins/index.d.ts index 236cc9d1d1..b1a4fa3ce5 100644 --- a/src/plugins/index.d.ts +++ b/src/plugins/index.d.ts @@ -1,4 +1,4 @@ -import Objects from './liveobjects'; +import Objects from './objects'; import Push from './push'; export interface StandardPlugins { diff --git a/src/plugins/liveobjects/batchcontext.ts b/src/plugins/objects/batchcontext.ts similarity index 96% rename from src/plugins/liveobjects/batchcontext.ts rename to src/plugins/objects/batchcontext.ts index 5fad94d6df..a20de010ee 100644 --- a/src/plugins/liveobjects/batchcontext.ts +++ b/src/plugins/objects/batchcontext.ts @@ -4,8 +4,8 @@ import { BatchContextLiveCounter } from './batchcontextlivecounter'; import { BatchContextLiveMap } from './batchcontextlivemap'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; -import { Objects } from './liveobjects'; -import { ROOT_OBJECT_ID } from './liveobjectspool'; +import { Objects } from './objects'; +import { ROOT_OBJECT_ID } from './objectspool'; import { StateMessage } from './statemessage'; export class BatchContext { diff --git a/src/plugins/liveobjects/batchcontextlivecounter.ts b/src/plugins/objects/batchcontextlivecounter.ts similarity index 96% rename from src/plugins/liveobjects/batchcontextlivecounter.ts rename to src/plugins/objects/batchcontextlivecounter.ts index 63ecaee09b..6c82129356 100644 --- a/src/plugins/liveobjects/batchcontextlivecounter.ts +++ b/src/plugins/objects/batchcontextlivecounter.ts @@ -1,7 +1,7 @@ import type BaseClient from 'common/lib/client/baseclient'; import { BatchContext } from './batchcontext'; import { LiveCounter } from './livecounter'; -import { Objects } from './liveobjects'; +import { Objects } from './objects'; export class BatchContextLiveCounter { private _client: BaseClient; diff --git a/src/plugins/liveobjects/batchcontextlivemap.ts b/src/plugins/objects/batchcontextlivemap.ts similarity index 98% rename from src/plugins/liveobjects/batchcontextlivemap.ts rename to src/plugins/objects/batchcontextlivemap.ts index 7971b17555..951caaa818 100644 --- a/src/plugins/liveobjects/batchcontextlivemap.ts +++ b/src/plugins/objects/batchcontextlivemap.ts @@ -2,7 +2,7 @@ import type * as API from '../../../ably'; import { BatchContext } from './batchcontext'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; -import { Objects } from './liveobjects'; +import { Objects } from './objects'; export class BatchContextLiveMap { constructor( diff --git a/src/plugins/liveobjects/defaults.ts b/src/plugins/objects/defaults.ts similarity index 100% rename from src/plugins/liveobjects/defaults.ts rename to src/plugins/objects/defaults.ts diff --git a/src/plugins/liveobjects/index.ts b/src/plugins/objects/index.ts similarity index 76% rename from src/plugins/liveobjects/index.ts rename to src/plugins/objects/index.ts index 11ed1762b2..411bee5e5f 100644 --- a/src/plugins/liveobjects/index.ts +++ b/src/plugins/objects/index.ts @@ -1,4 +1,4 @@ -import { Objects } from './liveobjects'; +import { Objects } from './objects'; import { StateMessage } from './statemessage'; export { Objects, StateMessage }; diff --git a/src/plugins/liveobjects/livecounter.ts b/src/plugins/objects/livecounter.ts similarity index 99% rename from src/plugins/liveobjects/livecounter.ts rename to src/plugins/objects/livecounter.ts index 216266e9fa..074c4a3f06 100644 --- a/src/plugins/liveobjects/livecounter.ts +++ b/src/plugins/objects/livecounter.ts @@ -1,6 +1,6 @@ import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; -import { Objects } from './liveobjects'; import { ObjectId } from './objectid'; +import { Objects } from './objects'; import { StateCounterOp, StateMessage, StateObject, StateOperation, StateOperationAction } from './statemessage'; export interface LiveCounterData extends LiveObjectData { diff --git a/src/plugins/liveobjects/livemap.ts b/src/plugins/objects/livemap.ts similarity index 99% rename from src/plugins/liveobjects/livemap.ts rename to src/plugins/objects/livemap.ts index 11178f232c..5271de6549 100644 --- a/src/plugins/liveobjects/livemap.ts +++ b/src/plugins/objects/livemap.ts @@ -3,8 +3,8 @@ import deepEqual from 'deep-equal'; import type * as API from '../../../ably'; import { DEFAULTS } from './defaults'; import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; -import { Objects } from './liveobjects'; import { ObjectId } from './objectid'; +import { Objects } from './objects'; import { MapSemantics, StateMapEntry, diff --git a/src/plugins/liveobjects/liveobject.ts b/src/plugins/objects/liveobject.ts similarity index 99% rename from src/plugins/liveobjects/liveobject.ts rename to src/plugins/objects/liveobject.ts index 55aa44b03f..85944f9c18 100644 --- a/src/plugins/liveobjects/liveobject.ts +++ b/src/plugins/objects/liveobject.ts @@ -1,6 +1,6 @@ import type BaseClient from 'common/lib/client/baseclient'; import type EventEmitter from 'common/lib/util/eventemitter'; -import { Objects } from './liveobjects'; +import { Objects } from './objects'; import { StateMessage, StateObject, StateOperation } from './statemessage'; export enum LiveObjectSubscriptionEvent { diff --git a/src/plugins/liveobjects/objectid.ts b/src/plugins/objects/objectid.ts similarity index 100% rename from src/plugins/liveobjects/objectid.ts rename to src/plugins/objects/objectid.ts diff --git a/src/plugins/liveobjects/liveobjects.ts b/src/plugins/objects/objects.ts similarity index 99% rename from src/plugins/liveobjects/liveobjects.ts rename to src/plugins/objects/objects.ts index c1de3e7d8d..282a411171 100644 --- a/src/plugins/liveobjects/liveobjects.ts +++ b/src/plugins/objects/objects.ts @@ -7,9 +7,9 @@ import { DEFAULTS } from './defaults'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; -import { ObjectsPool, ROOT_OBJECT_ID } from './liveobjectspool'; +import { ObjectsPool, ROOT_OBJECT_ID } from './objectspool'; import { StateMessage, StateOperationAction } from './statemessage'; -import { SyncObjectsDataPool } from './syncliveobjectsdatapool'; +import { SyncObjectsDataPool } from './syncobjectsdatapool'; export enum ObjectsEvent { syncing = 'syncing', diff --git a/src/plugins/liveobjects/liveobjectspool.ts b/src/plugins/objects/objectspool.ts similarity index 98% rename from src/plugins/liveobjects/liveobjectspool.ts rename to src/plugins/objects/objectspool.ts index 81485236e0..30d7c43d1b 100644 --- a/src/plugins/liveobjects/liveobjectspool.ts +++ b/src/plugins/objects/objectspool.ts @@ -3,8 +3,8 @@ import { DEFAULTS } from './defaults'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject } from './liveobject'; -import { Objects } from './liveobjects'; import { ObjectId } from './objectid'; +import { Objects } from './objects'; export const ROOT_OBJECT_ID = 'root'; diff --git a/src/plugins/liveobjects/statemessage.ts b/src/plugins/objects/statemessage.ts similarity index 100% rename from src/plugins/liveobjects/statemessage.ts rename to src/plugins/objects/statemessage.ts diff --git a/src/plugins/liveobjects/syncliveobjectsdatapool.ts b/src/plugins/objects/syncobjectsdatapool.ts similarity index 98% rename from src/plugins/liveobjects/syncliveobjectsdatapool.ts rename to src/plugins/objects/syncobjectsdatapool.ts index 6c41e9045a..2805efe2e2 100644 --- a/src/plugins/liveobjects/syncliveobjectsdatapool.ts +++ b/src/plugins/objects/syncobjectsdatapool.ts @@ -1,6 +1,6 @@ import type BaseClient from 'common/lib/client/baseclient'; import type RealtimeChannel from 'common/lib/client/realtimechannel'; -import { Objects } from './liveobjects'; +import { Objects } from './objects'; import { StateMessage, StateObject } from './statemessage'; export interface LiveObjectDataEntry { diff --git a/test/common/globals/named_dependencies.js b/test/common/globals/named_dependencies.js index 88b4f0e992..71075f7664 100644 --- a/test/common/globals/named_dependencies.js +++ b/test/common/globals/named_dependencies.js @@ -28,8 +28,8 @@ define(function () { node: 'test/common/modules/private_api_recorder', }, objects_helper: { - browser: 'test/common/modules/live_objects_helper', - node: 'test/common/modules/live_objects_helper', + browser: 'test/common/modules/objects_helper', + node: 'test/common/modules/objects_helper', }, }); }); diff --git a/test/common/modules/live_objects_helper.js b/test/common/modules/objects_helper.js similarity index 100% rename from test/common/modules/live_objects_helper.js rename to test/common/modules/objects_helper.js diff --git a/test/package/browser/template/README.md b/test/package/browser/template/README.md index e42f76296c..cc5cbc54c2 100644 --- a/test/package/browser/template/README.md +++ b/test/package/browser/template/README.md @@ -8,7 +8,7 @@ This directory is intended to be used for testing the following aspects of the a It contains three files, each of which import ably-js in different manners, and provide a way to briefly exercise its functionality: - `src/index-default.ts` imports the default ably-js package (`import { Realtime } from 'ably'`). -- `src/index-liveobjects.ts` imports the Objects ably-js plugin (`import Objects from 'ably/objects'`). +- `src/index-objects.ts` imports the Objects ably-js plugin (`import Objects from 'ably/objects'`). - `src/index-modular.ts` imports the tree-shakable ably-js package (`import { BaseRealtime, WebSocketTransport, FetchRequest } from 'ably/modular'`). - `src/ReactApp.tsx` imports React hooks from the ably-js package (`import { useChannel } from 'ably/react'`). @@ -26,7 +26,7 @@ This directory exposes three package scripts that are to be used for testing: - `build`: Uses esbuild to create: 1. a bundle containing `src/index-default.ts` and ably-js; - 2. a bundle containing `src/index-liveobjects.ts` and ably-js. + 2. a bundle containing `src/index-objects.ts` and ably-js. 3. a bundle containing `src/index-modular.ts` and ably-js. - `test`: Using the bundles created by `build` and playwright components setup, tests that the code that exercises ably-js’s functionality is working correctly in a browser. - `typecheck`: Type-checks the code that imports ably-js. diff --git a/test/package/browser/template/package.json b/test/package/browser/template/package.json index f2c023b6e6..574da1a7b3 100644 --- a/test/package/browser/template/package.json +++ b/test/package/browser/template/package.json @@ -4,7 +4,7 @@ "description": "", "main": "index.js", "scripts": { - "build": "esbuild --bundle src/index-default.ts --outdir=dist && esbuild --bundle src/index-liveobjects.ts --outdir=dist && esbuild --bundle src/index-modular.ts --outdir=dist", + "build": "esbuild --bundle src/index-default.ts --outdir=dist && esbuild --bundle src/index-objects.ts --outdir=dist && esbuild --bundle src/index-modular.ts --outdir=dist", "typecheck": "tsc --project src -noEmit", "test-support:server": "ts-node server/server.ts", "test": "npm run test:lib && npm run test:hooks", diff --git a/test/package/browser/template/server/resources/index-liveobjects.html b/test/package/browser/template/server/resources/index-objects.html similarity index 76% rename from test/package/browser/template/server/resources/index-liveobjects.html rename to test/package/browser/template/server/resources/index-objects.html index be37a0bb26..44d594e83c 100644 --- a/test/package/browser/template/server/resources/index-liveobjects.html +++ b/test/package/browser/template/server/resources/index-objects.html @@ -5,7 +5,7 @@ Ably NPM package test (Objects plugin export) - + diff --git a/test/package/browser/template/server/server.ts b/test/package/browser/template/server/server.ts index faa1399f70..0cac0b7f18 100644 --- a/test/package/browser/template/server/server.ts +++ b/test/package/browser/template/server/server.ts @@ -5,7 +5,7 @@ async function startWebServer(listenPort: number) { const server = express(); server.get('/', (req, res) => res.send('OK')); server.use(express.static(path.join(__dirname, '/resources'))); - for (const filename of ['index-default.js', 'index-liveobjects.js', 'index-modular.js']) { + for (const filename of ['index-default.js', 'index-objects.js', 'index-modular.js']) { server.use(`/${filename}`, express.static(path.join(__dirname, '..', 'dist', filename))); } diff --git a/test/package/browser/template/src/index-liveobjects.ts b/test/package/browser/template/src/index-objects.ts similarity index 100% rename from test/package/browser/template/src/index-liveobjects.ts rename to test/package/browser/template/src/index-objects.ts diff --git a/test/package/browser/template/test/lib/package.test.ts b/test/package/browser/template/test/lib/package.test.ts index 159b1e0c34..6c903d7f4a 100644 --- a/test/package/browser/template/test/lib/package.test.ts +++ b/test/package/browser/template/test/lib/package.test.ts @@ -3,7 +3,7 @@ import { test, expect } from '@playwright/test'; test.describe('NPM package', () => { for (const scenario of [ { name: 'default export', path: '/index-default.html' }, - { name: 'Objects plugin export', path: '/index-liveobjects.html' }, + { name: 'Objects plugin export', path: '/index-objects.html' }, { name: 'modular export', path: '/index-modular.html' }, ]) { test.describe(scenario.name, () => { diff --git a/test/realtime/live_objects.test.js b/test/realtime/objects.test.js similarity index 100% rename from test/realtime/live_objects.test.js rename to test/realtime/objects.test.js diff --git a/test/support/browser_file_list.js b/test/support/browser_file_list.js index d49cbee9e7..a305fac440 100644 --- a/test/support/browser_file_list.js +++ b/test/support/browser_file_list.js @@ -39,7 +39,7 @@ window.__testFiles__.files = { 'test/realtime/failure.test.js': true, 'test/realtime/history.test.js': true, 'test/realtime/init.test.js': true, - 'test/realtime/live_objects.test.js': true, + 'test/realtime/objects.test.js': true, 'test/realtime/message.test.js': true, 'test/realtime/presence.test.js': true, 'test/realtime/reauth.test.js': true, From 1aa89e3a4c753a46e125e601b05c68d09fa4113a Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 20 Mar 2025 08:55:20 +0000 Subject: [PATCH 137/166] Update bitwise value for `HAS_STATE` flag `HAS_STATE` flag was to moved to attach flags in [1]. [1] https://github.com/ably/realtime/pull/7230 Resolves PUB-1529 --- src/common/lib/types/protocolmessage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index 455f25105c..b154bf0b71 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -50,6 +50,7 @@ const flags: { [key: string]: number } = { RESUMED: 1 << 2, TRANSIENT: 1 << 4, ATTACH_RESUME: 1 << 5, + HAS_STATE: 1 << 7, /* Channel mode flags */ PRESENCE: 1 << 16, PUBLISH: 1 << 17, @@ -57,7 +58,6 @@ const flags: { [key: string]: number } = { PRESENCE_SUBSCRIBE: 1 << 19, OBJECT_SUBSCRIBE: 1 << 24, OBJECT_PUBLISH: 1 << 25, - HAS_STATE: 1 << 26, }; const flagNames = Object.keys(flags); flags.MODE_ALL = From bb41379911584d538cf5328c7bec34be7c6257dc Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 26 Mar 2025 09:21:00 +0000 Subject: [PATCH 138/166] Change Counter update object to have `amount` property instead of `inc` `amount` is more suitable in this case as the value can be negative in case the counter was decremented. Resolves PUB-1540 --- README.md | 4 ++-- ably.d.ts | 2 +- src/plugins/objects/livecounter.ts | 8 ++++---- test/package/browser/template/src/index-objects.ts | 2 +- test/realtime/objects.test.js | 8 ++++---- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 562be82385..085cc2a0ee 100644 --- a/README.md +++ b/README.md @@ -747,11 +747,11 @@ await root.remove('name'); await counter.increment(5); // LiveCounter new value: 5 -// LiveCounter update details: { update: { inc: 5 } } +// LiveCounter update details: { update: { amount: 5 } } await counter.decrement(2); // LiveCounter new value: 3 -// LiveCounter update details: { update: { inc: -2 } } +// LiveCounter update details: { update: { amount: -2 } } ``` You can deregister subscription listeners as follows: diff --git a/ably.d.ts b/ably.d.ts index a2b44643ea..72fbee441d 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -2415,7 +2415,7 @@ export declare interface LiveCounterUpdate extends LiveObjectUpdate { /** * The value by which the counter was incremented or decremented. */ - inc: number; + amount: number; }; } diff --git a/src/plugins/objects/livecounter.ts b/src/plugins/objects/livecounter.ts index 074c4a3f06..0706d9162a 100644 --- a/src/plugins/objects/livecounter.ts +++ b/src/plugins/objects/livecounter.ts @@ -8,7 +8,7 @@ export interface LiveCounterData extends LiveObjectData { } export interface LiveCounterUpdate extends LiveObjectUpdate { - update: { inc: number }; + update: { amount: number }; } export class LiveCounter extends LiveObject { @@ -291,7 +291,7 @@ export class LiveCounter extends LiveObject protected _updateFromDataDiff(prevDataRef: LiveCounterData, newDataRef: LiveCounterData): LiveCounterUpdate { const counterDiff = newDataRef.data - prevDataRef.data; - return { update: { inc: counterDiff } }; + return { update: { amount: counterDiff } }; } protected _mergeInitialDataFromCreateOperation(stateOperation: StateOperation): LiveCounterUpdate { @@ -302,7 +302,7 @@ export class LiveCounter extends LiveObject this._dataRef.data += stateOperation.counter?.count ?? 0; this._createOperationIsMerged = true; - return { update: { inc: stateOperation.counter?.count ?? 0 } }; + return { update: { amount: stateOperation.counter?.count ?? 0 } }; } private _throwNoPayloadError(op: StateOperation): void { @@ -332,6 +332,6 @@ export class LiveCounter extends LiveObject private _applyCounterInc(op: StateCounterOp): LiveCounterUpdate { this._dataRef.data += op.amount; - return { update: { inc: op.amount } }; + return { update: { amount: op.amount } }; } } diff --git a/test/package/browser/template/src/index-objects.ts b/test/package/browser/template/src/index-objects.ts index 269a9214a1..fa8e0c1707 100644 --- a/test/package/browser/template/src/index-objects.ts +++ b/test/package/browser/template/src/index-objects.ts @@ -57,7 +57,7 @@ globalThis.testAblyPackage = async function () { // same deal with nullish coalescing const value: number = counter?.value()!; const counterSubscribeResponse = counter?.subscribe(({ update }) => { - const shouldBeANumber: number = update.inc; + const shouldBeANumber: number = update.amount; }); counterSubscribeResponse?.unsubscribe(); diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js index 83348b17e0..116ffd2ab7 100644 --- a/test/realtime/objects.test.js +++ b/test/realtime/objects.test.js @@ -778,7 +778,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function root.get('counter').subscribe((update) => { try { expect(update).to.deep.equal( - { update: { inc: -1 } }, + { update: { amount: -1 } }, 'Check counter subscription callback is called with an expected update object after STATE_SYNC sequence with "tombstone=true"', ); resolve(); @@ -1776,7 +1776,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function root.get('counter').subscribe((update) => { try { expect(update).to.deep.equal( - { update: { inc: -1 } }, + { update: { amount: -1 } }, 'Check counter subscription callback is called with an expected update object after OBJECT_DELETE operation', ); resolve(); @@ -3567,7 +3567,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function counter.subscribe((update) => { try { expect(update).to.deep.equal( - { update: { inc: 1 } }, + { update: { amount: 1 } }, 'Check counter subscription callback is called with an expected update object for COUNTER_INC operation', ); resolve(); @@ -3604,7 +3604,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function try { const expectedInc = expectedCounterIncrements[currentUpdateIndex]; expect(update).to.deep.equal( - { update: { inc: expectedInc } }, + { update: { amount: expectedInc } }, `Check counter subscription callback is called with an expected update object for ${currentUpdateIndex + 1} times`, ); From 55977ddade7b83ff056d32f55cab684e6b9e5d87 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 27 Mar 2025 10:38:44 +0000 Subject: [PATCH 139/166] Replace internal usage of "state" term with "objects" for LiveObjects feature This commit does next changes: - removes `state` mentions in comments / internal API. Uses `objects` instead where possible - normalizes terminology used when referencing "LiveObjects" product and objects on a channel according to LiveObjects product docs PR [1] - `STATE` message action -> `OBJECT` - `STATE_SYNC` message action -> `OBJECT_SYNC` - `HAS_STATE` flag -> `HAS_OBJECT` - `StateMessage` type -> `ObjectMessage` - `StateOperation` type -> `ObjectOperation` - `StateObject` type -> `ObjectState` - `StateData` type -> `ObjectData` - `StateValue` type -> `ObjectValue` This brings ably-js LiveObjects implementation in line with the naming changes introduced in the spec PR [2]. [1] https://github.com/ably/docs/pull/2463#pullrequestreview-2678358626 [2] https://github.com/ably/specification/pull/279#discussion_r2003003310 --- README.md | 28 +- ably.d.ts | 39 +- src/common/lib/client/realtimechannel.ts | 42 +- src/common/lib/transport/protocol.ts | 2 +- src/common/lib/types/protocolmessage.ts | 19 +- src/plugins/objects/batchcontext.ts | 12 +- .../objects/batchcontextlivecounter.ts | 4 +- src/plugins/objects/batchcontextlivemap.ts | 8 +- src/plugins/objects/index.ts | 6 +- src/plugins/objects/livecounter.ts | 114 ++--- src/plugins/objects/livemap.ts | 240 +++++----- src/plugins/objects/liveobject.ts | 28 +- src/plugins/objects/objects.ts | 130 +++--- src/plugins/objects/statemessage.ts | 286 ++++++------ src/plugins/objects/syncobjectsdatapool.ts | 36 +- test/common/modules/objects_helper.js | 26 +- test/common/modules/private_api_recorder.js | 6 +- .../browser/template/src/index-objects.ts | 2 +- test/realtime/channel.test.js | 4 +- test/realtime/objects.test.js | 423 +++++++++--------- 20 files changed, 735 insertions(+), 720 deletions(-) diff --git a/README.md b/README.md index 085cc2a0ee..fc4401bb32 100644 --- a/README.md +++ b/README.md @@ -645,7 +645,7 @@ The authentication token must include corresponding capabilities for the client #### Getting the Root Object -The root object represents the top-level entry point for Objects within a channel. It gives access to all other nested Live Objects. +The root object represents the top-level entry point for objects within a channel. It gives access to all other nested objects. ```typescript const root = await objects.getRoot(); @@ -653,11 +653,11 @@ const root = await objects.getRoot(); The root object is a `LiveMap` instance and serves as the starting point for storing and organizing Objects on a channel. -#### Live Object Types +#### Object Types LiveObjects currently supports two primary data structures; `LiveMap` and `LiveCounter`. -`LiveMap` - A key/value map data structure, similar to a JavaScript `Map`, where all changes are synchronized across clients in realtime. It allows you to store primitive values and other Live Objects, enabling composability. +`LiveMap` - A key/value map data structure, similar to a JavaScript `Map`, where all changes are synchronized across clients in realtime. It enables you to store primitive values and other objects, enabling composability. You can use `LiveMap` as follows: @@ -689,7 +689,7 @@ await root.set('foo', 'Alice'); await root.set('bar', 1); await root.set('baz', true); await root.set('qux', new Uint8Array([21, 31])); -// as well as other live objects +// as well as other objects const counter = await objects.createCounter(); await root.set('quux', counter); @@ -714,11 +714,11 @@ await counter.decrement(2); #### Subscribing to Updates -Subscribing to updates on Live Objects allows you to receive changes made by other clients in realtime. Since multiple clients may modify the same Live Objects, subscribing ensures that your application reacts to external updates as soon as they are received. +Subscribing to updates on objects enables you to receive changes made by other clients in realtime. Since multiple clients may modify the same objects, subscribing ensures that your application reacts to external updates as soon as they are received. Additionally, mutation methods such as `LiveMap.set`, `LiveCounter.increment`, and `LiveCounter.decrement` do not directly edit the current state of the object locally. Instead, they send the intended operation to the Ably system, and the change is applied to the local object only when the corresponding realtime operation is echoed back to the client. This means that the state you retrieve immediately after a mutation may not reflect the latest updates yet. -You can subscribe to updates on all Live Objects using subscription listeners as follows: +You can subscribe to updates on all objects using subscription listeners as follows: ```typescript const root = await objects.getRoot(); @@ -772,14 +772,14 @@ root.unsubscribeAll(); #### Creating New Objects -New `LiveMap` and `LiveCounter` instances can be created as follows: +New `LiveMap` and `LiveCounter` objects can be created as follows: ```typescript const counter = await objects.createCounter(123); // with optional initial counter value const map = await objects.createMap({ key: 'value' }); // with optional initial map entries ``` -To persist them within the Objects state, they must be assigned to a parent `LiveMap` that is connected to the root object through the object hierarchy: +To persist them on a channel and share them between clients, they must be assigned to a parent `LiveMap` that is connected to the root object through the object hierarchy: ```typescript const root = await objects.getRoot(); @@ -799,9 +799,9 @@ await root.set('outerMap', outerMap); #### Batch Operations -Batching allows multiple operations to be grouped into a single message that is sent to the Ably service. This allows batched operations to be applied atomically together. +Batching enables multiple operations to be grouped into a single channel message that is sent to the Ably service. This guarantees that all changes are applied atomically. -Within a batch callback, the `BatchContext` instance provides wrapper objects around regular Live Objects with a synchronous API for storing changes in the batch context. +Within a batch callback, the `BatchContext` instance provides wrapper objects around regular `LiveMap` and `LiveCounter` objects with a synchronous API for storing changes in the batch context. ```typescript await objects.batch((ctx) => { @@ -819,9 +819,9 @@ await objects.batch((ctx) => { #### Lifecycle Events -Live Objects emit lifecycle events to signal critical state changes, such as synchronization progress and object deletions. +LiveObjects emit events that allow you to monitor objects' lifecycle changes, such as synchronization progress and object deletions. -**Synchronization Events** - the `syncing` and `synced` events notify when the Objects state is being synchronized with the Ably service. These events can be useful for displaying loading indicators, preventing user edits during synchronization, or triggering application logic when the data was loaded for the first time. +**Synchronization Events** - the `syncing` and `synced` events notify when the local Objects state on a client is being synchronized with the Ably service. These events can be useful for displaying loading indicators, preventing user edits during synchronization, or triggering application logic when the data was loaded for the first time. ```typescript objects.on('syncing', () => { @@ -835,14 +835,14 @@ objects.on('synced', () => { }); ``` -**Object Deletion Events** - objects that have been orphaned for a long period (i.e., not connected to the state tree graph by being set as a key in a map accessible from the root map object) will eventually be deleted. Once a Live Object is deleted, it can no longer be interacted with. You should avoid accessing its data or trying to update its value and you should remove all references to the deleted object in your application. +**Object Deletion Events** - objects that have been orphaned for a long period (i.e., not connected to the object tree by being set as a key in a map accessible from the root map object) will eventually be deleted. Once an object is deleted, it can no longer be interacted with. You should avoid accessing its data or trying to update its value and you should remove all references to the deleted object in your application. ```typescript const root = await objects.getRoot(); const counter = root.get('counter'); counter.on('deleted', () => { - console.log('Live Object has been deleted.'); + console.log('Object has been deleted.'); // Example: Remove references to the object from the application }); ``` diff --git a/ably.d.ts b/ably.d.ts index 72fbee441d..3b48903d1c 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -872,11 +872,11 @@ declare namespace ChannelModes { */ type PRESENCE_SUBSCRIBE = 'PRESENCE_SUBSCRIBE'; /** - * The client can publish Objects messages. + * The client can publish object messages. */ type OBJECT_PUBLISH = 'OBJECT_PUBLISH'; /** - * The client can receive Objects messages. + * The client can receive object messages. */ type OBJECT_SUBSCRIBE = 'OBJECT_SUBSCRIBE'; /** @@ -1560,9 +1560,9 @@ export type DeregisterCallback = (device: DeviceDetails, callback: StandardCallb export type ErrorCallback = (error: ErrorInfo | null) => void; /** - * A callback used in {@link LiveObject} to listen for updates to the Live Object. + * A callback used in {@link LiveObject} to listen for updates to the object. * - * @param update - The update object describing the changes made to the Live Object. + * @param update - The update object describing the changes made to the object. */ export type LiveObjectUpdateCallback = (update: T) => void; @@ -2181,9 +2181,9 @@ declare global { /** * Represents the type of data stored in a {@link LiveMap}. - * It maps string keys to scalar values ({@link StateValue}), or other Live Objects. + * It maps string keys to scalar values ({@link ObjectValue}), or other {@link LiveObject | LiveObjects}. */ -export type LiveMapType = { [key: string]: StateValue | LiveMap | LiveCounter | undefined }; +export type LiveMapType = { [key: string]: ObjectValue | LiveMap | LiveCounter | undefined }; /** * The default type for the `root` object for Objects on a channel, based on the globally defined {@link ObjectsTypes} interface. @@ -2236,7 +2236,7 @@ export declare interface BatchContextLiveMap { get(key: TKey): T[TKey] | undefined; /** - * Returns the number of key/value pairs in the map. + * Returns the number of key-value pairs in the map. */ size(): number; @@ -2293,10 +2293,11 @@ export declare interface BatchContextLiveCounter { } /** - * The `LiveMap` class represents a key/value map data structure, similar to a JavaScript Map, where all changes are synchronized across clients in realtime. - * Conflict-free resolution for updates follows Last Write Wins (LWW) semantics, meaning that if two clients update the same key in the map, the update with the most recent timestamp wins. + * The `LiveMap` class represents a key-value map data structure, similar to a JavaScript Map, where all changes are synchronized across clients in realtime. + * Conflicts in a LiveMap are automatically resolved with last-write-wins (LWW) semantics, + * meaning that if two clients update the same key in the map, the update with the most recent timestamp wins. * - * Keys must be strings. Values can be another Live Object, or a primitive type, such as a string, number, boolean, or binary data (see {@link StateValue}). + * Keys must be strings. Values can be another {@link LiveObject}, or a primitive type, such as a string, number, boolean, or binary data (see {@link ObjectValue}). */ export declare interface LiveMap extends LiveObject { /** @@ -2310,12 +2311,12 @@ export declare interface LiveMap extends LiveObject(key: TKey): T[TKey] | undefined; /** - * Returns the number of key/value pairs in the map. + * Returns the number of key-value pairs in the map. */ size(): number; /** - * Returns an iterable of key/value pairs for every entry in the map. + * Returns an iterable of key-value pairs for every entry in the map. */ entries(): IterableIterator<[TKey, T[TKey]]>; @@ -2372,7 +2373,7 @@ export declare interface LiveMapUpdate extends LiveObjectUpdate { * * For binary data, the resulting type depends on the platform (`Buffer` in Node.js, `ArrayBuffer` elsewhere). */ -export type StateValue = string | number | boolean | Buffer | ArrayBuffer; +export type ObjectValue = string | number | boolean | Buffer | ArrayBuffer; /** * The `LiveCounter` class represents a counter that can be incremented or decremented and is synchronized across clients in realtime. @@ -2424,22 +2425,22 @@ export declare interface LiveCounterUpdate extends LiveObjectUpdate { */ export declare interface LiveObject { /** - * Registers a listener that is called each time this Live Object is updated. + * Registers a listener that is called each time this LiveObject is updated. * - * @param listener - An event listener function that is called with an update object whenever this Live Object is updated. + * @param listener - An event listener function that is called with an update object whenever this LiveObject is updated. * @returns A {@link SubscribeResponse} object that allows the provided listener to be deregistered from future updates. */ subscribe(listener: LiveObjectUpdateCallback): SubscribeResponse; /** - * Deregisters the given listener from updates for this Live Object. + * Deregisters the given listener from updates for this LiveObject. * * @param listener - An event listener function. */ unsubscribe(listener: LiveObjectUpdateCallback): void; /** - * Deregisters all listeners from updates for this Live Object. + * Deregisters all listeners from updates for this LiveObject. */ unsubscribeAll(): void; @@ -2467,7 +2468,7 @@ export declare interface LiveObject { + sendState(objectMessages: ObjectMessage[]): Promise { return new Promise((resolve, reject) => { const msg = protocolMessageFromValues({ - action: actions.STATE, + action: actions.OBJECT, channel: this.name, - state, + state: objectMessages, }); this.sendMessage(msg, (err) => (err ? reject(err) : resolve())); }); @@ -524,7 +524,7 @@ class RealtimeChannel extends EventEmitter { message.action === actions.ATTACHED || message.action === actions.MESSAGE || message.action === actions.PRESENCE || - message.action === actions.STATE + message.action === actions.OBJECT ) { // RTL15b this.setChannelSerial(message.channelSerial); @@ -542,7 +542,7 @@ class RealtimeChannel extends EventEmitter { const resumed = message.hasFlag('RESUMED'); const hasPresence = message.hasFlag('HAS_PRESENCE'); const hasBacklog = message.hasFlag('HAS_BACKLOG'); - const hasState = message.hasFlag('HAS_STATE'); + const hasObjects = message.hasFlag('HAS_OBJECTS'); if (this.state === 'attached') { if (!resumed) { // we have lost continuity. @@ -550,9 +550,9 @@ class RealtimeChannel extends EventEmitter { if (this._presence) { this._presence.onAttached(hasPresence); } - // the Objects state needs to be re-synced + // the Objects tree needs to be re-synced if (this._objects) { - this._objects.onAttached(hasState); + this._objects.onAttached(hasObjects); } } const change = new ChannelStateChange(this.state, this.state, resumed, hasBacklog, message.error); @@ -564,7 +564,7 @@ class RealtimeChannel extends EventEmitter { /* RTL5i: re-send DETACH and remain in the 'detaching' state */ this.checkPendingState(); } else { - this.notifyState('attached', message.error, resumed, hasPresence, hasBacklog, hasState); + this.notifyState('attached', message.error, resumed, hasPresence, hasBacklog, hasObjects); } break; } @@ -612,25 +612,25 @@ class RealtimeChannel extends EventEmitter { break; } - // STATE and STATE_SYNC message processing share most of the logic, so group them together - case actions.STATE: - case actions.STATE_SYNC: { + // OBJECT and OBJECT_SYNC message processing share most of the logic, so group them together + case actions.OBJECT: + case actions.OBJECT_SYNC: { if (!this._objects) { return; } - const stateMessages = message.state ?? []; + const objectMessages = message.state ?? []; const options = this.channelOptions; - await this._decodeAndPrepareMessages(message, stateMessages, (msg) => + await this._decodeAndPrepareMessages(message, objectMessages, (msg) => this.client._objectsPlugin - ? this.client._objectsPlugin.StateMessage.decode(msg, options, MessageEncoding) + ? this.client._objectsPlugin.ObjectMessage.decode(msg, options, MessageEncoding) : Utils.throwMissingPluginError('Objects'), ); - if (message.action === actions.STATE) { - this._objects.handleStateMessages(stateMessages); + if (message.action === actions.OBJECT) { + this._objects.handleObjectMessages(objectMessages); } else { - this._objects.handleStateSyncMessages(stateMessages, message.channelSerial); + this._objects.handleObjectSyncMessages(objectMessages, message.channelSerial); } break; @@ -740,7 +740,7 @@ class RealtimeChannel extends EventEmitter { * @returns `unrecoverableError` flag. If `true` indicates that unrecoverable error was encountered during message decoding * and any further message processing should be stopped. Always equals to `false` if `decodeErrorRecoveryHandler` was not provided */ - private async _decodeAndPrepareMessages( + private async _decodeAndPrepareMessages( protocolMessage: ProtocolMessage, messages: T[], decodeFn: (msg: T) => Promise, @@ -809,7 +809,7 @@ class RealtimeChannel extends EventEmitter { resumed?: boolean, hasPresence?: boolean, hasBacklog?: boolean, - hasState?: boolean, + hasObjects?: boolean, ): void { Logger.logAction( this.logger, @@ -831,7 +831,7 @@ class RealtimeChannel extends EventEmitter { this._presence.actOnChannelState(state, hasPresence, reason); } if (this._objects) { - this._objects.actOnChannelState(state, hasState); + this._objects.actOnChannelState(state, hasObjects); } if (state === 'suspended' && this.connectionManager.state.sendEvents) { this.startRetryTimer(); diff --git a/src/common/lib/transport/protocol.ts b/src/common/lib/transport/protocol.ts index 05d50da966..9b8095326f 100644 --- a/src/common/lib/transport/protocol.ts +++ b/src/common/lib/transport/protocol.ts @@ -21,7 +21,7 @@ export class PendingMessage { const action = message.action; this.sendAttempted = false; this.ackRequired = - typeof action === 'number' && [actions.MESSAGE, actions.PRESENCE, actions.STATE].includes(action); + typeof action === 'number' && [actions.MESSAGE, actions.PRESENCE, actions.OBJECT].includes(action); } } diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index b154bf0b71..148c3c3350 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -34,8 +34,8 @@ export const actions = { SYNC: 16, AUTH: 17, ACTIVATE: 18, - STATE: 19, - STATE_SYNC: 20, + OBJECT: 19, + OBJECT_SYNC: 20, }; export const ActionName: string[] = []; @@ -50,7 +50,7 @@ const flags: { [key: string]: number } = { RESUMED: 1 << 2, TRANSIENT: 1 << 4, ATTACH_RESUME: 1 << 5, - HAS_STATE: 1 << 7, + HAS_OBJECTS: 1 << 7, /* Channel mode flags */ PRESENCE: 1 << 16, PUBLISH: 1 << 17, @@ -126,12 +126,12 @@ export function fromDeserialized( } } - let state: ObjectsPlugin.StateMessage[] | undefined = undefined; + let state: ObjectsPlugin.ObjectMessage[] | undefined = undefined; if (objectsPlugin) { - state = deserialized.state as ObjectsPlugin.StateMessage[]; + state = deserialized.state as ObjectsPlugin.ObjectMessage[]; if (state) { for (let i = 0; i < state.length; i++) { - state[i] = objectsPlugin.StateMessage.fromValues(state[i], Utils, MessageEncoding); + state[i] = objectsPlugin.ObjectMessage.fromValues(state[i], Utils, MessageEncoding); } } } @@ -143,7 +143,7 @@ export function fromDeserialized( * Used internally by the tests. * * ObjectsPlugin code can't be included as part of the core library to prevent size growth, - * so if a test needs to build state messages, then it must provide the plugin upon call. + * so if a test needs to build object messages, then it must provide the plugin upon call. */ export function makeFromDeserializedWithDependencies(dependencies?: { ObjectsPlugin: typeof ObjectsPlugin | null }) { return (deserialized: Record): ProtocolMessage => { @@ -178,7 +178,8 @@ export function stringify( if (msg.presence && presenceMessagePlugin) result += '; presence=' + toStringArray(presenceMessagePlugin.presenceMessagesFromValuesArray(msg.presence)); if (msg.state && objectsPlugin) { - result += '; state=' + toStringArray(objectsPlugin.StateMessage.fromValuesArray(msg.state, Utils, MessageEncoding)); + result += + '; state=' + toStringArray(objectsPlugin.ObjectMessage.fromValuesArray(msg.state, Utils, MessageEncoding)); } if (msg.error) result += '; error=' + ErrorInfo.fromValues(msg.error).toString(); if (msg.auth && msg.auth.accessToken) result += '; token=' + msg.auth.accessToken; @@ -218,7 +219,7 @@ class ProtocolMessage { /** * This will be undefined if we skipped decoding this property due to user not requesting Objects functionality — see {@link fromDeserialized} */ - state?: ObjectsPlugin.StateMessage[]; + state?: ObjectsPlugin.ObjectMessage[]; auth?: unknown; connectionDetails?: Record; diff --git a/src/plugins/objects/batchcontext.ts b/src/plugins/objects/batchcontext.ts index a20de010ee..8194bba034 100644 --- a/src/plugins/objects/batchcontext.ts +++ b/src/plugins/objects/batchcontext.ts @@ -6,13 +6,13 @@ import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { Objects } from './objects'; import { ROOT_OBJECT_ID } from './objectspool'; -import { StateMessage } from './statemessage'; +import { ObjectMessage } from './statemessage'; export class BatchContext { private _client: BaseClient; - /** Maps live object ids to the corresponding batch context object wrappers */ + /** Maps object ids to the corresponding batch context object wrappers */ private _wrappedObjects: Map> = new Map(); - private _queuedMessages: StateMessage[] = []; + private _queuedMessages: ObjectMessage[] = []; private _isClosed = false; constructor( @@ -49,7 +49,7 @@ export class BatchContext { wrappedObject = new BatchContextLiveCounter(this, this._objects, originObject); } else { throw new this._client.ErrorInfo( - `Unknown Live Object instance type: objectId=${originObject.getObjectId()}`, + `Unknown LiveObject instance type: objectId=${originObject.getObjectId()}`, 50000, 500, ); @@ -85,8 +85,8 @@ export class BatchContext { /** * @internal */ - queueStateMessage(stateMessage: StateMessage): void { - this._queuedMessages.push(stateMessage); + queueMessage(msg: ObjectMessage): void { + this._queuedMessages.push(msg); } /** diff --git a/src/plugins/objects/batchcontextlivecounter.ts b/src/plugins/objects/batchcontextlivecounter.ts index 6c82129356..7f7b6ac1d3 100644 --- a/src/plugins/objects/batchcontextlivecounter.ts +++ b/src/plugins/objects/batchcontextlivecounter.ts @@ -23,8 +23,8 @@ export class BatchContextLiveCounter { increment(amount: number): void { this._objects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); - const stateMessage = LiveCounter.createCounterIncMessage(this._objects, this._counter.getObjectId(), amount); - this._batchContext.queueStateMessage(stateMessage); + const msg = LiveCounter.createCounterIncMessage(this._objects, this._counter.getObjectId(), amount); + this._batchContext.queueMessage(msg); } decrement(amount: number): void { diff --git a/src/plugins/objects/batchcontextlivemap.ts b/src/plugins/objects/batchcontextlivemap.ts index 951caaa818..306b313426 100644 --- a/src/plugins/objects/batchcontextlivemap.ts +++ b/src/plugins/objects/batchcontextlivemap.ts @@ -49,14 +49,14 @@ export class BatchContextLiveMap { set(key: TKey, value: T[TKey]): void { this._objects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); - const stateMessage = LiveMap.createMapSetMessage(this._objects, this._map.getObjectId(), key, value); - this._batchContext.queueStateMessage(stateMessage); + const msg = LiveMap.createMapSetMessage(this._objects, this._map.getObjectId(), key, value); + this._batchContext.queueMessage(msg); } remove(key: TKey): void { this._objects.throwIfInvalidWriteApiConfiguration(); this._batchContext.throwIfClosed(); - const stateMessage = LiveMap.createMapRemoveMessage(this._objects, this._map.getObjectId(), key); - this._batchContext.queueStateMessage(stateMessage); + const msg = LiveMap.createMapRemoveMessage(this._objects, this._map.getObjectId(), key); + this._batchContext.queueMessage(msg); } } diff --git a/src/plugins/objects/index.ts b/src/plugins/objects/index.ts index 411bee5e5f..e9144588c5 100644 --- a/src/plugins/objects/index.ts +++ b/src/plugins/objects/index.ts @@ -1,9 +1,9 @@ import { Objects } from './objects'; -import { StateMessage } from './statemessage'; +import { ObjectMessage } from './statemessage'; -export { Objects, StateMessage }; +export { Objects, ObjectMessage }; export default { Objects, - StateMessage, + ObjectMessage, }; diff --git a/src/plugins/objects/livecounter.ts b/src/plugins/objects/livecounter.ts index 0706d9162a..8cc8781f6f 100644 --- a/src/plugins/objects/livecounter.ts +++ b/src/plugins/objects/livecounter.ts @@ -1,7 +1,7 @@ import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { ObjectId } from './objectid'; import { Objects } from './objects'; -import { StateCounterOp, StateMessage, StateObject, StateOperation, StateOperationAction } from './statemessage'; +import { CounterOp, ObjectMessage, ObjectOperation, ObjectOperationAction, ObjectState } from './statemessage'; export interface LiveCounterData extends LiveObjectData { data: number; @@ -22,58 +22,58 @@ export class LiveCounter extends LiveObject } /** - * Returns a {@link LiveCounter} instance based on the provided state object. - * The provided state object must hold a valid counter object data. + * Returns a {@link LiveCounter} instance based on the provided object state. + * The provided object state must hold a valid counter object data. * * @internal */ - static fromStateObject(objects: Objects, stateObject: StateObject): LiveCounter { - const obj = new LiveCounter(objects, stateObject.objectId); - obj.overrideWithStateObject(stateObject); + static fromObjectState(objects: Objects, objectState: ObjectState): LiveCounter { + const obj = new LiveCounter(objects, objectState.objectId); + obj.overrideWithObjectState(objectState); return obj; } /** - * Returns a {@link LiveCounter} instance based on the provided COUNTER_CREATE state operation. - * The provided state operation must hold a valid counter object data. + * Returns a {@link LiveCounter} instance based on the provided COUNTER_CREATE object operation. + * The provided object operation must hold a valid counter object data. * * @internal */ - static fromStateOperation(objects: Objects, stateOperation: StateOperation): LiveCounter { - const obj = new LiveCounter(objects, stateOperation.objectId); - obj._mergeInitialDataFromCreateOperation(stateOperation); + static fromObjectOperation(objects: Objects, objectOperation: ObjectOperation): LiveCounter { + const obj = new LiveCounter(objects, objectOperation.objectId); + obj._mergeInitialDataFromCreateOperation(objectOperation); return obj; } /** * @internal */ - static createCounterIncMessage(objects: Objects, objectId: string, amount: number): StateMessage { + static createCounterIncMessage(objects: Objects, objectId: string, amount: number): ObjectMessage { const client = objects.getClient(); - if (typeof amount !== 'number' || !isFinite(amount)) { + if (typeof amount !== 'number' || !Number.isFinite(amount)) { throw new client.ErrorInfo('Counter value increment should be a valid number', 40003, 400); } - const stateMessage = StateMessage.fromValues( + const msg = ObjectMessage.fromValues( { operation: { - action: StateOperationAction.COUNTER_INC, + action: ObjectOperationAction.COUNTER_INC, objectId, counterOp: { amount }, - } as StateOperation, + } as ObjectOperation, }, client.Utils, client.MessageEncoding, ); - return stateMessage; + return msg; } /** * @internal */ - static async createCounterCreateMessage(objects: Objects, count?: number): Promise { + static async createCounterCreateMessage(objects: Objects, count?: number): Promise { const client = objects.getClient(); if (count !== undefined && (typeof count !== 'number' || !Number.isFinite(count))) { @@ -81,7 +81,7 @@ export class LiveCounter extends LiveObject } const initialValueObj = LiveCounter.createInitialValueObject(count); - const { encodedInitialValue, format } = StateMessage.encodeInitialValue(initialValueObj, client); + const { encodedInitialValue, format } = ObjectMessage.encodeInitialValue(initialValueObj, client); const nonce = client.Utils.cheapRandStr(); const msTimestamp = await client.getTimestamp(true); @@ -93,28 +93,28 @@ export class LiveCounter extends LiveObject msTimestamp, ).toString(); - const stateMessage = StateMessage.fromValues( + const msg = ObjectMessage.fromValues( { operation: { ...initialValueObj, - action: StateOperationAction.COUNTER_CREATE, + action: ObjectOperationAction.COUNTER_CREATE, objectId, nonce, initialValue: encodedInitialValue, initialValueEncoding: format, - } as StateOperation, + } as ObjectOperation, }, client.Utils, client.MessageEncoding, ); - return stateMessage; + return msg; } /** * @internal */ - static createInitialValueObject(count?: number): Pick { + static createInitialValueObject(count?: number): Pick { return { counter: { count: count ?? 0, @@ -138,8 +138,8 @@ export class LiveCounter extends LiveObject */ async increment(amount: number): Promise { this._objects.throwIfInvalidWriteApiConfiguration(); - const stateMessage = LiveCounter.createCounterIncMessage(this._objects, this.getObjectId(), amount); - return this._objects.publish([stateMessage]); + const msg = LiveCounter.createCounterIncMessage(this._objects, this.getObjectId(), amount); + return this._objects.publish([msg]); } /** @@ -149,7 +149,7 @@ export class LiveCounter extends LiveObject this._objects.throwIfInvalidWriteApiConfiguration(); // do an explicit type safety check here before negating the amount value, // so we don't unintentionally change the type sent by a user - if (typeof amount !== 'number' || !isFinite(amount)) { + if (typeof amount !== 'number' || !Number.isFinite(amount)) { throw new this._client.ErrorInfo('Counter value decrement should be a valid number', 40003, 400); } @@ -159,10 +159,10 @@ export class LiveCounter extends LiveObject /** * @internal */ - applyOperation(op: StateOperation, msg: StateMessage): void { + applyOperation(op: ObjectOperation, msg: ObjectMessage): void { if (op.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( - `Cannot apply state operation with objectId=${op.objectId}, to this LiveCounter with objectId=${this.getObjectId()}`, + `Cannot apply object operation with objectId=${op.objectId}, to this LiveCounter with objectId=${this.getObjectId()}`, 92000, 500, ); @@ -190,11 +190,11 @@ export class LiveCounter extends LiveObject let update: LiveCounterUpdate | LiveObjectUpdateNoop; switch (op.action) { - case StateOperationAction.COUNTER_CREATE: + case ObjectOperationAction.COUNTER_CREATE: update = this._applyCounterCreate(op); break; - case StateOperationAction.COUNTER_INC: + case ObjectOperationAction.COUNTER_INC: if (this._client.Utils.isNil(op.counterOp)) { this._throwNoPayloadError(op); // leave an explicit return here, so that TS knows that update object is always set after the switch statement. @@ -204,7 +204,7 @@ export class LiveCounter extends LiveObject } break; - case StateOperationAction.OBJECT_DELETE: + case ObjectOperationAction.OBJECT_DELETE: update = this._applyObjectDelete(); break; @@ -222,28 +222,28 @@ export class LiveCounter extends LiveObject /** * @internal */ - overrideWithStateObject(stateObject: StateObject): LiveCounterUpdate | LiveObjectUpdateNoop { - if (stateObject.objectId !== this.getObjectId()) { + overrideWithObjectState(objectState: ObjectState): LiveCounterUpdate | LiveObjectUpdateNoop { + if (objectState.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( - `Invalid state object: state object objectId=${stateObject.objectId}; LiveCounter objectId=${this.getObjectId()}`, + `Invalid object state: object state objectId=${objectState.objectId}; LiveCounter objectId=${this.getObjectId()}`, 92000, 500, ); } - if (!this._client.Utils.isNil(stateObject.createOp)) { - // it is expected that create operation can be missing in the state object, so only validate it when it exists - if (stateObject.createOp.objectId !== this.getObjectId()) { + if (!this._client.Utils.isNil(objectState.createOp)) { + // it is expected that create operation can be missing in the object state, so only validate it when it exists + if (objectState.createOp.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( - `Invalid state object: state object createOp objectId=${stateObject.createOp?.objectId}; LiveCounter objectId=${this.getObjectId()}`, + `Invalid object state: object state createOp objectId=${objectState.createOp?.objectId}; LiveCounter objectId=${this.getObjectId()}`, 92000, 500, ); } - if (stateObject.createOp.action !== StateOperationAction.COUNTER_CREATE) { + if (objectState.createOp.action !== ObjectOperationAction.COUNTER_CREATE) { throw new this._client.ErrorInfo( - `Invalid state object: state object createOp action=${stateObject.createOp?.action}; LiveCounter objectId=${this.getObjectId()}`, + `Invalid object state: object state createOp action=${objectState.createOp?.action}; LiveCounter objectId=${this.getObjectId()}`, 92000, 500, ); @@ -251,29 +251,29 @@ export class LiveCounter extends LiveObject } // object's site timeserials are still updated even if it is tombstoned, so always use the site timeserials received from the op. - // should default to empty map if site timeserials do not exist on the state object, so that any future operation may be applied to this object. - this._siteTimeserials = stateObject.siteTimeserials ?? {}; + // should default to empty map if site timeserials do not exist on the object state, so that any future operation may be applied to this object. + this._siteTimeserials = objectState.siteTimeserials ?? {}; if (this.isTombstoned()) { - // this object is tombstoned. this is a terminal state which can't be overridden. skip the rest of state object message processing + // this object is tombstoned. this is a terminal state which can't be overridden. skip the rest of object state message processing return { noop: true }; } const previousDataRef = this._dataRef; - if (stateObject.tombstone) { - // tombstone this object and ignore the data from the state object message + if (objectState.tombstone) { + // tombstone this object and ignore the data from the object state message this.tombstone(); } else { - // override data for this object with data from the state object + // override data for this object with data from the object state this._createOperationIsMerged = false; - this._dataRef = { data: stateObject.counter?.count ?? 0 }; - if (!this._client.Utils.isNil(stateObject.createOp)) { - this._mergeInitialDataFromCreateOperation(stateObject.createOp); + this._dataRef = { data: objectState.counter?.count ?? 0 }; + if (!this._client.Utils.isNil(objectState.createOp)) { + this._mergeInitialDataFromCreateOperation(objectState.createOp); } } // if object got tombstoned, the update object will include all data that got cleared. - // otherwise it is a diff between previous value and new value from state object. + // otherwise it is a diff between previous value and new value from object state. return this._updateFromDataDiff(previousDataRef, this._dataRef); } @@ -294,18 +294,18 @@ export class LiveCounter extends LiveObject return { update: { amount: counterDiff } }; } - protected _mergeInitialDataFromCreateOperation(stateOperation: StateOperation): LiveCounterUpdate { + protected _mergeInitialDataFromCreateOperation(objectOperation: ObjectOperation): LiveCounterUpdate { // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly 0 in this case. // note that it is intentional to SUM the incoming count from the create op. // if we got here, it means that current counter instance is missing the initial value in its data reference, // which we're going to add now. - this._dataRef.data += stateOperation.counter?.count ?? 0; + this._dataRef.data += objectOperation.counter?.count ?? 0; this._createOperationIsMerged = true; - return { update: { amount: stateOperation.counter?.count ?? 0 } }; + return { update: { amount: objectOperation.counter?.count ?? 0 } }; } - private _throwNoPayloadError(op: StateOperation): void { + private _throwNoPayloadError(op: ObjectOperation): void { throw new this._client.ErrorInfo( `No payload found for ${op.action} op for LiveCounter objectId=${this.getObjectId()}`, 92000, @@ -313,7 +313,7 @@ export class LiveCounter extends LiveObject ); } - private _applyCounterCreate(op: StateOperation): LiveCounterUpdate | LiveObjectUpdateNoop { + private _applyCounterCreate(op: ObjectOperation): LiveCounterUpdate | LiveObjectUpdateNoop { if (this._createOperationIsMerged) { // There can't be two different create operation for the same object id, because the object id // fully encodes that operation. This means we can safely ignore any new incoming create operations @@ -330,7 +330,7 @@ export class LiveCounter extends LiveObject return this._mergeInitialDataFromCreateOperation(op); } - private _applyCounterInc(op: StateCounterOp): LiveCounterUpdate { + private _applyCounterInc(op: CounterOp): LiveCounterUpdate { this._dataRef.data += op.amount; return { update: { amount: op.amount } }; } diff --git a/src/plugins/objects/livemap.ts b/src/plugins/objects/livemap.ts index 5271de6549..415e309b92 100644 --- a/src/plugins/objects/livemap.ts +++ b/src/plugins/objects/livemap.ts @@ -7,44 +7,44 @@ import { ObjectId } from './objectid'; import { Objects } from './objects'; import { MapSemantics, - StateMapEntry, - StateMapOp, - StateMessage, - StateObject, - StateOperation, - StateOperationAction, - StateValue, + MapEntry, + MapOp, + ObjectMessage, + ObjectState, + ObjectOperation, + ObjectOperationAction, + ObjectValue, } from './statemessage'; -export interface ObjectIdStateData { - /** A reference to another state object, used to support composable state objects. */ +export interface ObjectIdObjectData { + /** A reference to another object, used to support composable object structures. */ objectId: string; } -export interface ValueStateData { +export interface ValueObjectData { /** * The encoding the client should use to interpret the value. * Analogous to the `encoding` field on the `Message` and `PresenceMessage` types. */ encoding?: string; - /** A concrete leaf value in the state object graph. */ - value: StateValue; + /** A concrete leaf value in the object graph. */ + value: ObjectValue; } -export type StateData = ObjectIdStateData | ValueStateData; +export type ObjectData = ObjectIdObjectData | ValueObjectData; -export interface MapEntry { +export interface LiveMapEntry { tombstone: boolean; /** * Can't use timeserial from the operation that deleted the entry for the same reason as for {@link LiveObject} tombstones, see explanation there. */ tombstonedAt: number | undefined; timeserial: string | undefined; - data: StateData | undefined; + data: ObjectData | undefined; } export interface LiveMapData extends LiveObjectData { - data: Map; + data: Map; } export interface LiveMapUpdate extends LiveObjectUpdate { @@ -70,26 +70,29 @@ export class LiveMap extends LiveObject(objects: Objects, stateObject: StateObject): LiveMap { - const obj = new LiveMap(objects, stateObject.map?.semantics!, stateObject.objectId); - obj.overrideWithStateObject(stateObject); + static fromObjectState(objects: Objects, objectState: ObjectState): LiveMap { + const obj = new LiveMap(objects, objectState.map?.semantics!, objectState.objectId); + obj.overrideWithObjectState(objectState); return obj; } /** - * Returns a {@link LiveMap} instance based on the provided MAP_CREATE state operation. - * The provided state operation must hold a valid map object data. + * Returns a {@link LiveMap} instance based on the provided MAP_CREATE object operation. + * The provided object operation must hold a valid map object data. * * @internal */ - static fromStateOperation(objects: Objects, stateOperation: StateOperation): LiveMap { - const obj = new LiveMap(objects, stateOperation.map?.semantics!, stateOperation.objectId); - obj._mergeInitialDataFromCreateOperation(stateOperation); + static fromObjectOperation( + objects: Objects, + objectOperation: ObjectOperation, + ): LiveMap { + const obj = new LiveMap(objects, objectOperation.map?.semantics!, objectOperation.objectId); + obj._mergeInitialDataFromCreateOperation(objectOperation); return obj; } @@ -101,32 +104,32 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject { + static async createMapCreateMessage(objects: Objects, entries?: API.LiveMapType): Promise { const client = objects.getClient(); if (entries !== undefined && (entries === null || typeof entries !== 'object')) { - throw new client.ErrorInfo('Map entries should be a key/value object', 40003, 400); + throw new client.ErrorInfo('Map entries should be a key-value object', 40003, 400); } Object.entries(entries ?? {}).forEach(([key, value]) => LiveMap.validateKeyValue(objects, key, value)); const initialValueObj = LiveMap.createInitialValueObject(entries); - const { encodedInitialValue, format } = StateMessage.encodeInitialValue(initialValueObj, client); + const { encodedInitialValue, format } = ObjectMessage.encodeInitialValue(initialValueObj, client); const nonce = client.Utils.cheapRandStr(); const msTimestamp = await client.getTimestamp(true); @@ -208,45 +211,45 @@ export class LiveMap extends LiveObject { - const stateMapEntries: Record = {}; + static createInitialValueObject(entries?: API.LiveMapType): Pick { + const mapEntries: Record = {}; Object.entries(entries ?? {}).forEach(([key, value]) => { - const stateData: StateData = + const objectData: ObjectData = value instanceof LiveObject - ? ({ objectId: value.getObjectId() } as ObjectIdStateData) - : ({ value } as ValueStateData); + ? ({ objectId: value.getObjectId() } as ObjectIdObjectData) + : ({ value } as ValueObjectData); - stateMapEntries[key] = { - data: stateData, + mapEntries[key] = { + data: objectData, }; }); return { map: { semantics: MapSemantics.LWW, - entries: stateMapEntries, + entries: mapEntries, }, }; } @@ -257,7 +260,7 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject extends LiveObject(key: TKey, value: T[TKey]): Promise { this._objects.throwIfInvalidWriteApiConfiguration(); - const stateMessage = LiveMap.createMapSetMessage(this._objects, this.getObjectId(), key, value); - return this._objects.publish([stateMessage]); + const msg = LiveMap.createMapSetMessage(this._objects, this.getObjectId(), key, value); + return this._objects.publish([msg]); } /** @@ -352,17 +355,17 @@ export class LiveMap extends LiveObject(key: TKey): Promise { this._objects.throwIfInvalidWriteApiConfiguration(); - const stateMessage = LiveMap.createMapRemoveMessage(this._objects, this.getObjectId(), key); - return this._objects.publish([stateMessage]); + const msg = LiveMap.createMapRemoveMessage(this._objects, this.getObjectId(), key); + return this._objects.publish([msg]); } /** * @internal */ - applyOperation(op: StateOperation, msg: StateMessage): void { + applyOperation(op: ObjectOperation, msg: ObjectMessage): void { if (op.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( - `Cannot apply state operation with objectId=${op.objectId}, to this LiveMap with objectId=${this.getObjectId()}`, + `Cannot apply object operation with objectId=${op.objectId}, to this LiveMap with objectId=${this.getObjectId()}`, 92000, 500, ); @@ -390,11 +393,11 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject() }; + return { data: new Map() }; } protected _updateFromDataDiff(prevDataRef: LiveMapData, newDataRef: LiveMapData): LiveMapUpdate { @@ -577,8 +580,8 @@ export class LiveMap extends LiveObject extends LiveObject { + Object.entries(objectOperation.map.entries ?? {}).forEach(([key, entry]) => { // for MAP_CREATE op we must use dedicated timeserial field available on an entry, instead of a timeserial on a message const opOriginTimeserial = entry.timeserial; let update: LiveMapUpdate | LiveObjectUpdateNoop; @@ -613,7 +616,7 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject entryTimeserial; } - private _liveMapDataFromMapEntries(entries: Record): LiveMapData { + private _liveMapDataFromMapEntries(entries: Record): LiveMapData { const liveMapData: LiveMapData = { - data: new Map(), + data: new Map(), }; - // need to iterate over entries manually to work around optional parameters from state object entries type + // need to iterate over entries to correctly process optional parameters Object.entries(entries ?? {}).forEach(([key, entry]) => { - let liveData: StateData; + let liveData: ObjectData; if (typeof entry.data.objectId !== 'undefined') { - liveData = { objectId: entry.data.objectId } as ObjectIdStateData; + liveData = { objectId: entry.data.objectId } as ObjectIdObjectData; } else { - liveData = { encoding: entry.data.encoding, value: entry.data.value } as ValueStateData; + liveData = { encoding: entry.data.encoding, value: entry.data.value } as ValueObjectData; } - const liveDataEntry: MapEntry = { + const liveDataEntry: LiveMapEntry = { timeserial: entry.timeserial, data: liveData, // consider object as tombstoned only if we received an explicit flag stating that. otherwise it exists @@ -789,15 +789,15 @@ export class LiveMap extends LiveObject extends LiveObject; @@ -155,7 +155,7 @@ export abstract class LiveObject< } /** - * Clears the object's state, cancels any buffered operations and sets the tombstone flag to `true`. + * Clears the object's data, cancels any buffered operations and sets the tombstone flag to `true`. * * @internal */ @@ -206,24 +206,24 @@ export abstract class LiveObject< } /** - * Apply state operation message on live object. + * Apply object operation message on this LiveObject. * * @internal */ - abstract applyOperation(op: StateOperation, msg: StateMessage): void; + abstract applyOperation(op: ObjectOperation, msg: ObjectMessage): void; /** - * Overrides internal data for live object with data from the given state object. - * Provided state object should hold a valid data for current live object, e.g. counter data for LiveCounter, map data for LiveMap. + * Overrides internal data for this LiveObject with data from the given object state. + * Provided object state should hold a valid data for current LiveObject, e.g. counter data for LiveCounter, map data for LiveMap. * - * State objects are received during SYNC sequence, and SYNC sequence is a source of truth for the current state of the objects, - * so we can use the data received from the SYNC sequence directly and override any data values or site timeserials this live object has + * Object states are received during sync sequence, and sync sequence is a source of truth for the current state of the objects, + * so we can use the data received from the sync sequence directly and override any data values or site timeserials this LiveObject has * without the need to merge them. * * Returns an update object that describes the changes applied based on the object's previous value. * * @internal */ - abstract overrideWithStateObject(stateObject: StateObject): TUpdate | LiveObjectUpdateNoop; + abstract overrideWithObjectState(objectState: ObjectState): TUpdate | LiveObjectUpdateNoop; /** * @internal */ @@ -231,18 +231,18 @@ export abstract class LiveObject< protected abstract _getZeroValueData(): TData; /** - * Calculate the update object based on the current Live Object data and incoming new data. + * Calculate the update object based on the current LiveObject data and incoming new data. */ protected abstract _updateFromDataDiff(prevDataRef: TData, newDataRef: TData): TUpdate; /** - * Merges the initial data from the create operation into the live object. + * Merges the initial data from the create operation into the LiveObject. * - * Client SDKs do not need to keep around the state operation that created the object, + * Client SDKs do not need to keep around the object operation that created the object, * so we can merge the initial data the first time we receive it for the object, * and work with aggregated value after that. * * This saves us from needing to merge the initial value with operations applied to * the object every time the object is read. */ - protected abstract _mergeInitialDataFromCreateOperation(stateOperation: StateOperation): TUpdate; + protected abstract _mergeInitialDataFromCreateOperation(objectOperation: ObjectOperation): TUpdate; } diff --git a/src/plugins/objects/objects.ts b/src/plugins/objects/objects.ts index 282a411171..a21401702b 100644 --- a/src/plugins/objects/objects.ts +++ b/src/plugins/objects/objects.ts @@ -8,7 +8,7 @@ import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { ObjectsPool, ROOT_OBJECT_ID } from './objectspool'; -import { StateMessage, StateOperationAction } from './statemessage'; +import { ObjectMessage, ObjectOperationAction } from './statemessage'; import { SyncObjectsDataPool } from './syncobjectsdatapool'; export enum ObjectsEvent { @@ -49,7 +49,7 @@ export class Objects { private _syncObjectsDataPool: SyncObjectsDataPool; private _currentSyncId: string | undefined; private _currentSyncCursor: string | undefined; - private _bufferedStateOperations: StateMessage[]; + private _bufferedObjectOperations: ObjectMessage[]; // Used by tests static _DEFAULTS = DEFAULTS; @@ -62,7 +62,7 @@ export class Objects { this._eventEmitterPublic = new this._client.EventEmitter(this._client.logger); this._objectsPool = new ObjectsPool(this); this._syncObjectsDataPool = new SyncObjectsDataPool(this); - this._bufferedStateOperations = []; + this._bufferedObjectOperations = []; } /** @@ -73,7 +73,7 @@ export class Objects { async getRoot(): Promise> { this.throwIfInvalidAccessApiConfiguration(); - // if we're not synced yet, wait for SYNC sequence to finish before returning root + // if we're not synced yet, wait for sync sequence to finish before returning root if (this._state !== ObjectsState.synced) { await this._eventEmitterInternal.once(ObjectsEvent.synced); } @@ -109,10 +109,10 @@ export class Objects { async createMap(entries?: T): Promise> { this.throwIfInvalidWriteApiConfiguration(); - const stateMessage = await LiveMap.createMapCreateMessage(this, entries); - const objectId = stateMessage.operation?.objectId!; + const msg = await LiveMap.createMapCreateMessage(this, entries); + const objectId = msg.operation?.objectId!; - await this.publish([stateMessage]); + await this.publish([msg]); // we may have already received the CREATE operation at this point, as it could arrive before the ACK for our publish message. // this means the object might already exist in the local pool, having been added during the usual CREATE operation process. @@ -121,10 +121,10 @@ export class Objects { return this._objectsPool.get(objectId) as LiveMap; } - // we haven't received the CREATE operation yet, so we can create a new map object using the locally constructed state operation. + // we haven't received the CREATE operation yet, so we can create a new map object using the locally constructed object operation. // we don't know the timeserials for map entries, so we assign an "earliest possible" timeserial to each entry, so that any subsequent operation can be applied to them. // we mark the CREATE operation as merged for the object, guaranteeing its idempotency and preventing it from being applied again when the operation arrives. - const map = LiveMap.fromStateOperation(this, stateMessage.operation!); + const map = LiveMap.fromObjectOperation(this, msg.operation!); this._objectsPool.set(objectId, map); return map; @@ -141,10 +141,10 @@ export class Objects { async createCounter(count?: number): Promise { this.throwIfInvalidWriteApiConfiguration(); - const stateMessage = await LiveCounter.createCounterCreateMessage(this, count); - const objectId = stateMessage.operation?.objectId!; + const msg = await LiveCounter.createCounterCreateMessage(this, count); + const objectId = msg.operation?.objectId!; - await this.publish([stateMessage]); + await this.publish([msg]); // we may have already received the CREATE operation at this point, as it could arrive before the ACK for our publish message. // this means the object might already exist in the local pool, having been added during the usual CREATE operation process. @@ -153,9 +153,9 @@ export class Objects { return this._objectsPool.get(objectId) as LiveCounter; } - // we haven't received the CREATE operation yet, so we can create a new counter object using the locally constructed state operation. + // we haven't received the CREATE operation yet, so we can create a new counter object using the locally constructed object operation. // we mark the CREATE operation as merged for the object, guaranteeing its idempotency. this ensures we don't double count the initial counter value when the operation arrives. - const counter = LiveCounter.fromStateOperation(this, stateMessage.operation!); + const counter = LiveCounter.fromObjectOperation(this, msg.operation!); this._objectsPool.set(objectId, counter); return counter; @@ -212,14 +212,14 @@ export class Objects { /** * @internal */ - handleStateSyncMessages(stateMessages: StateMessage[], syncChannelSerial: string | null | undefined): void { + handleObjectSyncMessages(objectMessages: ObjectMessage[], syncChannelSerial: string | null | undefined): void { const { syncId, syncCursor } = this._parseSyncChannelSerial(syncChannelSerial); const newSyncSequence = this._currentSyncId !== syncId; if (newSyncSequence) { this._startNewSync(syncId, syncCursor); } - this._syncObjectsDataPool.applyStateSyncMessages(stateMessages); + this._syncObjectsDataPool.applyObjectSyncMessages(objectMessages); // if this is the last (or only) message in a sequence of sync updates, end the sync if (!syncCursor) { @@ -232,39 +232,39 @@ export class Objects { /** * @internal */ - handleStateMessages(stateMessages: StateMessage[]): void { + handleObjectMessages(objectMessages: ObjectMessage[]): void { if (this._state !== ObjectsState.synced) { - // The client receives state messages in realtime over the channel concurrently with the SYNC sequence. - // Some of the incoming state messages may have already been applied to the state objects described in - // the SYNC sequence, but others may not; therefore we must buffer these messages so that we can apply - // them to the state objects once the SYNC is complete. - this._bufferedStateOperations.push(...stateMessages); + // The client receives object messages in realtime over the channel concurrently with the sync sequence. + // Some of the incoming object messages may have already been applied to the objects described in + // the sync sequence, but others may not; therefore we must buffer these messages so that we can apply + // them to the objects once the sync is complete. + this._bufferedObjectOperations.push(...objectMessages); return; } - this._applyStateMessages(stateMessages); + this._applyObjectMessages(objectMessages); } /** * @internal */ - onAttached(hasState?: boolean): void { + onAttached(hasObjects?: boolean): void { this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MINOR, 'Objects.onAttached()', - `channel=${this._channel.name}, hasState=${hasState}`, + `channel=${this._channel.name}, hasObjects=${hasObjects}`, ); const fromInitializedState = this._state === ObjectsState.initialized; - if (hasState || fromInitializedState) { - // should always start a new sync sequence if we're in the initialized state, no matter the HAS_STATE flag value. + if (hasObjects || fromInitializedState) { + // should always start a new sync sequence if we're in the initialized state, no matter the HAS_OBJECTS flag value. // this guarantees we emit both "syncing" -> "synced" events in that order. this._startNewSync(); } - if (!hasState) { - // if no HAS_STATE flag received on attach, we can end SYNC sequence immediately and treat it as no state on a channel. + if (!hasObjects) { + // if no HAS_OBJECTS flag received on attach, we can end sync sequence immediately and treat it as no objects on a channel. this._objectsPool.reset(); this._syncObjectsDataPool.reset(); // defer the state change event until the next tick if we started a new sequence just now due to being in initialized state. @@ -276,10 +276,10 @@ export class Objects { /** * @internal */ - actOnChannelState(state: API.ChannelState, hasState?: boolean): void { + actOnChannelState(state: API.ChannelState, hasObjects?: boolean): void { switch (state) { case 'attached': - this.onAttached(hasState); + this.onAttached(hasObjects); break; case 'detached': @@ -293,7 +293,7 @@ export class Objects { /** * @internal */ - async publish(stateMessages: StateMessage[]): Promise { + async publish(objectMessages: ObjectMessage[]): Promise { if (!this._channel.connectionManager.activeState()) { throw this._channel.connectionManager.getError(); } @@ -302,18 +302,18 @@ export class Objects { throw this._client.ErrorInfo.fromValues(this._channel.invalidStateError()); } - stateMessages.forEach((x) => StateMessage.encode(x, this._client.MessageEncoding)); + objectMessages.forEach((x) => ObjectMessage.encode(x, this._client.MessageEncoding)); const maxMessageSize = this._client.options.maxMessageSize; - const size = stateMessages.reduce((acc, msg) => acc + msg.getMessageSize(), 0); + const size = objectMessages.reduce((acc, msg) => acc + msg.getMessageSize(), 0); if (size > maxMessageSize) { throw new this._client.ErrorInfo( - `Maximum size of state messages that can be published at once exceeded (was ${size} bytes; limit is ${maxMessageSize} bytes)`, + `Maximum size of object messages that can be published at once exceeded (was ${size} bytes; limit is ${maxMessageSize} bytes)`, 40009, 400, ); } - return this._channel.sendState(stateMessages); + return this._channel.sendState(objectMessages); } /** @@ -333,8 +333,8 @@ export class Objects { } private _startNewSync(syncId?: string, syncCursor?: string): void { - // need to discard all buffered state operation messages on new sync start - this._bufferedStateOperations = []; + // need to discard all buffered object operation messages on new sync start + this._bufferedObjectOperations = []; this._syncObjectsDataPool.reset(); this._currentSyncId = syncId; this._currentSyncCursor = syncCursor; @@ -343,11 +343,11 @@ export class Objects { private _endSync(deferStateEvent: boolean): void { this._applySync(); - // should apply buffered state operations after we applied the SYNC data. - // can use regular state messages application logic - this._applyStateMessages(this._bufferedStateOperations); + // should apply buffered object operations after we applied the sync. + // can use regular object messages application logic + this._applyObjectMessages(this._bufferedObjectOperations); - this._bufferedStateOperations = []; + this._bufferedObjectOperations = []; this._syncObjectsDataPool.reset(); this._currentSyncId = undefined; this._currentSyncCursor = undefined; @@ -385,8 +385,8 @@ export class Objects { const existingObject = this._objectsPool.get(objectId); if (existingObject) { - const update = existingObject.overrideWithStateObject(entry.stateObject); - // store updates to call subscription callbacks for all of them once the SYNC sequence is completed. + const update = existingObject.overrideWithObjectState(entry.objectState); + // store updates to call subscription callbacks for all of them once the sync sequence is completed. // this will ensure that clients get notified about the changes only once everything has been applied. existingObjectUpdates.push({ object: existingObject, update }); continue; @@ -397,64 +397,64 @@ export class Objects { const objectType = entry.objectType; switch (objectType) { case 'LiveCounter': - newObject = LiveCounter.fromStateObject(this, entry.stateObject); + newObject = LiveCounter.fromObjectState(this, entry.objectState); break; case 'LiveMap': - newObject = LiveMap.fromStateObject(this, entry.stateObject); + newObject = LiveMap.fromObjectState(this, entry.objectState); break; default: - throw new this._client.ErrorInfo(`Unknown Live Object type: ${objectType}`, 50000, 500); + throw new this._client.ErrorInfo(`Unknown LiveObject type: ${objectType}`, 50000, 500); } this._objectsPool.set(objectId, newObject); } - // need to remove Live Object instances from the ObjectsPool for which objectIds were not received during the SYNC sequence + // need to remove LiveObject instances from the ObjectsPool for which objectIds were not received during the sync sequence this._objectsPool.deleteExtraObjectIds([...receivedObjectIds]); // call subscription callbacks for all updated existing objects existingObjectUpdates.forEach(({ object, update }) => object.notifyUpdated(update)); } - private _applyStateMessages(stateMessages: StateMessage[]): void { - for (const stateMessage of stateMessages) { - if (!stateMessage.operation) { + private _applyObjectMessages(objectMessages: ObjectMessage[]): void { + for (const objectMessage of objectMessages) { + if (!objectMessage.operation) { this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MAJOR, - 'Objects._applyStateMessages()', - `state operation message is received without 'operation' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + 'Objects._applyObjectMessages()', + `object operation message is received without 'operation' field, skipping message; message id: ${objectMessage.id}, channel: ${this._channel.name}`, ); continue; } - const stateOperation = stateMessage.operation; + const objectOperation = objectMessage.operation; - switch (stateOperation.action) { - case StateOperationAction.MAP_CREATE: - case StateOperationAction.COUNTER_CREATE: - case StateOperationAction.MAP_SET: - case StateOperationAction.MAP_REMOVE: - case StateOperationAction.COUNTER_INC: - case StateOperationAction.OBJECT_DELETE: + switch (objectOperation.action) { + case ObjectOperationAction.MAP_CREATE: + case ObjectOperationAction.COUNTER_CREATE: + case ObjectOperationAction.MAP_SET: + case ObjectOperationAction.MAP_REMOVE: + case ObjectOperationAction.COUNTER_INC: + case ObjectOperationAction.OBJECT_DELETE: // we can receive an op for an object id we don't have yet in the pool. instead of buffering such operations, // we can create a zero-value object for the provided object id and apply the operation to that zero-value object. // this also means that all objects are capable of applying the corresponding *_CREATE ops on themselves, // since they need to be able to eventually initialize themselves from that *_CREATE op. // so to simplify operations handling, we always try to create a zero-value object in the pool first, // and then we can always apply the operation on the existing object in the pool. - this._objectsPool.createZeroValueObjectIfNotExists(stateOperation.objectId); - this._objectsPool.get(stateOperation.objectId)!.applyOperation(stateOperation, stateMessage); + this._objectsPool.createZeroValueObjectIfNotExists(objectOperation.objectId); + this._objectsPool.get(objectOperation.objectId)!.applyOperation(objectOperation, objectMessage); break; default: this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MAJOR, - 'Objects._applyStateMessages()', - `received unsupported action in state operation message: ${stateOperation.action}, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + 'Objects._applyObjectMessages()', + `received unsupported action in object operation message: ${objectOperation.action}, skipping message; message id: ${objectMessage.id}, channel: ${this._channel.name}`, ); } } diff --git a/src/plugins/objects/statemessage.ts b/src/plugins/objects/statemessage.ts index 8bceed1e24..f97cc912a1 100644 --- a/src/plugins/objects/statemessage.ts +++ b/src/plugins/objects/statemessage.ts @@ -6,7 +6,7 @@ import type { ChannelOptions } from 'common/types/channel'; export type EncodeFunction = (data: any, encoding?: string | null) => { data: any; encoding?: string | null }; -export enum StateOperationAction { +export enum ObjectOperationAction { MAP_CREATE = 0, MAP_SET = 1, MAP_REMOVE = 2, @@ -19,38 +19,38 @@ export enum MapSemantics { LWW = 0, } -/** A StateValue represents a concrete leaf value in a state object graph. */ -export type StateValue = string | number | boolean | Bufferlike; +/** An ObjectValue represents a concrete leaf value in the object graph. */ +export type ObjectValue = string | number | boolean | Bufferlike; -/** StateData captures a value in a state object. */ -export interface StateData { - /** A reference to another state object, used to support composable state objects. */ +/** An ObjectData represents a value in an object on a channel. */ +export interface ObjectData { + /** A reference to another object, used to support composable object structures. */ objectId?: string; /** * The encoding the client should use to interpret the value. * Analogous to the `encoding` field on the `Message` and `PresenceMessage` types. */ encoding?: string; - /** A concrete leaf value in the state object graph. */ - value?: StateValue; + /** A concrete leaf value in the object graph. */ + value?: ObjectValue; } -/** A StateMapOp describes an operation to be applied to a Map object. */ -export interface StateMapOp { +/** A MapOp describes an operation to be applied to a Map object. */ +export interface MapOp { /** The key of the map entry to which the operation should be applied. */ key: string; /** The data that the map entry should contain if the operation is a MAP_SET operation. */ - data?: StateData; + data?: ObjectData; } -/** A StateCounterOp describes an operation to be applied to a Counter object. */ -export interface StateCounterOp { +/** A CounterOp describes an operation to be applied to a Counter object. */ +export interface CounterOp { /** The data value that should be added to the counter */ amount: number; } /** A MapEntry represents the value at a given key in a Map object. */ -export interface StateMapEntry { +export interface MapEntry { /** Indicates whether the map entry has been removed. */ tombstone?: boolean; /** @@ -61,43 +61,43 @@ export interface StateMapEntry { */ timeserial?: string; /** The data that represents the value of the map entry. */ - data: StateData; + data: ObjectData; } -/** A Map object represents a map of key-value pairs. */ -export interface StateMap { +/** An ObjectMap object represents a map of key-value pairs. */ +export interface ObjectMap { /** The conflict-resolution semantics used by the map object. */ semantics?: MapSemantics; // The map entries, indexed by key. - entries?: Record; + entries?: Record; } -/** A Counter object represents an incrementable and decrementable value */ -export interface StateCounter { +/** An ObjectCounter object represents an incrementable and decrementable value */ +export interface ObjectCounter { /** The value of the counter */ count?: number; } -/** A StateOperation describes an operation to be applied to a state object. */ -export interface StateOperation { - /** Defines the operation to be applied to the state object. */ - action: StateOperationAction; - /** The object ID of the state object to which the operation should be applied. */ +/** An ObjectOperation describes an operation to be applied to an object on a channel. */ +export interface ObjectOperation { + /** Defines the operation to be applied to the object. */ + action: ObjectOperationAction; + /** The object ID of the object on a channel to which the operation should be applied. */ objectId: string; /** The payload for the operation if it is an operation on a Map object type. */ - mapOp?: StateMapOp; + mapOp?: MapOp; /** The payload for the operation if it is an operation on a Counter object type. */ - counterOp?: StateCounterOp; + counterOp?: CounterOp; /** * The payload for the operation if the operation is MAP_CREATE. - * Defines the initial value for the map object. + * Defines the initial value for the Map object. */ - map?: StateMap; + map?: ObjectMap; /** * The payload for the operation if the operation is COUNTER_CREATE. - * Defines the initial value for the counter object. + * Defines the initial value for the Counter object. */ - counter?: StateCounter; + counter?: ObjectCounter; /** * The nonce, must be present on create operations. This is the random part * that has been hashed with the type and initial value to create the object ID. @@ -114,47 +114,55 @@ export interface StateOperation { initialValueEncoding?: Utils.Format; } -/** A StateObject describes the instantaneous state of an object. */ -export interface StateObject { - /** The identifier of the state object. */ +/** An ObjectState describes the instantaneous state of an object on a channel. */ +export interface ObjectState { + /** The identifier of the object. */ objectId: string; - /** A vector of origin timeserials keyed by site code of the last operation that was applied to this state object. */ + /** A vector of origin timeserials keyed by site code of the last operation that was applied to this object. */ siteTimeserials: Record; /** True if the object has been tombstoned. */ tombstone: boolean; /** - * The operation that created the state object. + * The operation that created the object. * * Can be missing if create operation for the object is not known at this point. */ - createOp?: StateOperation; + createOp?: ObjectOperation; /** * The data that represents the result of applying all operations to a Map object * excluding the initial value from the create operation if it is a Map object type. */ - map?: StateMap; + map?: ObjectMap; /** * The data that represents the result of applying all operations to a Counter object * excluding the initial value from the create operation if it is a Counter object type. */ - counter?: StateCounter; + counter?: ObjectCounter; } /** * @internal */ -export class StateMessage { +export class ObjectMessage { id?: string; timestamp?: number; clientId?: string; connectionId?: string; channel?: string; extras?: any; - /** Describes an operation to be applied to a state object. */ - operation?: StateOperation; - /** Describes the instantaneous state of an object. */ - object?: StateObject; - /** Timeserial format. Contains the origin timeserial for this state message. */ + /** + * Describes an operation to be applied to an object. + * + * Mutually exclusive with the `object` field. This field is only set on object messages if the `action` field of the `ProtocolMessage` encapsulating it is `OBJECT`. + */ + operation?: ObjectOperation; + /** + * Describes the instantaneous state of an object. + * + * Mutually exclusive with the `operation` field. This field is only set on object messages if the `action` field of the `ProtocolMessage` encapsulating it is `OBJECT_SYNC`. + */ + object?: ObjectState; + /** Timeserial format. Contains the origin timeserial for this object message. */ serial?: string; /** Site code corresponding to this message's timeserial */ siteCode?: string; @@ -165,12 +173,12 @@ export class StateMessage { ) {} /** - * Protocol agnostic encoding of the state message's data entries. - * Mutates the provided StateMessage. + * Protocol agnostic encoding of the object message's data entries. + * Mutates the provided ObjectMessage. * * Uses encoding functions from regular `Message` processing. */ - static async encode(message: StateMessage, messageEncoding: typeof MessageEncoding): Promise { + static async encode(message: ObjectMessage, messageEncoding: typeof MessageEncoding): Promise { const encodeFn: EncodeFunction = (data, encoding) => { const { data: encodedData, encoding: newEncoding } = messageEncoding.encodeData(data, encoding); @@ -180,68 +188,70 @@ export class StateMessage { }; }; - message.operation = message.operation ? StateMessage._encodeStateOperation(message.operation, encodeFn) : undefined; - message.object = message.object ? StateMessage._encodeStateObject(message.object, encodeFn) : undefined; + message.operation = message.operation + ? ObjectMessage._encodeObjectOperation(message.operation, encodeFn) + : undefined; + message.object = message.object ? ObjectMessage._encodeObjectState(message.object, encodeFn) : undefined; return message; } /** - * Mutates the provided StateMessage and decodes all data entries in the message + * Mutates the provided ObjectMessage and decodes all data entries in the message */ static async decode( - message: StateMessage, + message: ObjectMessage, inputContext: ChannelOptions, messageEncoding: typeof MessageEncoding, ): Promise { // TODO: decide how to handle individual errors from decoding values. currently we throw first ever error we get if (message.object?.map?.entries) { - await StateMessage._decodeMapEntries(message.object.map.entries, inputContext, messageEncoding); + await ObjectMessage._decodeMapEntries(message.object.map.entries, inputContext, messageEncoding); } if (message.object?.createOp?.map?.entries) { - await StateMessage._decodeMapEntries(message.object.createOp.map.entries, inputContext, messageEncoding); + await ObjectMessage._decodeMapEntries(message.object.createOp.map.entries, inputContext, messageEncoding); } if (message.object?.createOp?.mapOp?.data && 'value' in message.object.createOp.mapOp.data) { - await StateMessage._decodeStateData(message.object.createOp.mapOp.data, inputContext, messageEncoding); + await ObjectMessage._decodeObjectData(message.object.createOp.mapOp.data, inputContext, messageEncoding); } if (message.operation?.map?.entries) { - await StateMessage._decodeMapEntries(message.operation.map.entries, inputContext, messageEncoding); + await ObjectMessage._decodeMapEntries(message.operation.map.entries, inputContext, messageEncoding); } if (message.operation?.mapOp?.data && 'value' in message.operation.mapOp.data) { - await StateMessage._decodeStateData(message.operation.mapOp.data, inputContext, messageEncoding); + await ObjectMessage._decodeObjectData(message.operation.mapOp.data, inputContext, messageEncoding); } } static fromValues( - values: StateMessage | Record, + values: ObjectMessage | Record, utils: typeof Utils, messageEncoding: typeof MessageEncoding, - ): StateMessage { - return Object.assign(new StateMessage(utils, messageEncoding), values); + ): ObjectMessage { + return Object.assign(new ObjectMessage(utils, messageEncoding), values); } static fromValuesArray( - values: (StateMessage | Record)[], + values: (ObjectMessage | Record)[], utils: typeof Utils, messageEncoding: typeof MessageEncoding, - ): StateMessage[] { + ): ObjectMessage[] { const count = values.length; const result = new Array(count); for (let i = 0; i < count; i++) { - result[i] = StateMessage.fromValues(values[i], utils, messageEncoding); + result[i] = ObjectMessage.fromValues(values[i], utils, messageEncoding); } return result; } static encodeInitialValue( - initialValue: Partial, + initialValue: Partial, client: BaseClient, ): { encodedInitialValue: Bufferlike; @@ -250,11 +260,11 @@ export class StateMessage { const format = client.options.useBinaryProtocol ? client.Utils.Format.msgpack : client.Utils.Format.json; // initial value object may contain user provided data that requires an additional encoding (for example buffers as map keys). - // so we need to encode that data first as if we were sending it over the wire. we can use a StateMessage methods for this - const stateMessage = StateMessage.fromValues({ operation: initialValue }, client.Utils, client.MessageEncoding); - StateMessage.encode(stateMessage, client.MessageEncoding); - const { operation: initialValueWithDataEncoding } = StateMessage._encodeForWireProtocol( - stateMessage, + // so we need to encode that data first as if we were sending it over the wire. we can use an ObjectMessage methods for this + const msg = ObjectMessage.fromValues({ operation: initialValue }, client.Utils, client.MessageEncoding); + ObjectMessage.encode(msg, client.MessageEncoding); + const { operation: initialValueWithDataEncoding } = ObjectMessage._encodeForWireProtocol( + msg, client.MessageEncoding, format, ); @@ -277,80 +287,80 @@ export class StateMessage { } private static async _decodeMapEntries( - mapEntries: Record, + mapEntries: Record, inputContext: ChannelOptions, messageEncoding: typeof MessageEncoding, ): Promise { for (const entry of Object.values(mapEntries)) { - await StateMessage._decodeStateData(entry.data, inputContext, messageEncoding); + await ObjectMessage._decodeObjectData(entry.data, inputContext, messageEncoding); } } - private static async _decodeStateData( - stateData: StateData, + private static async _decodeObjectData( + objectData: ObjectData, inputContext: ChannelOptions, messageEncoding: typeof MessageEncoding, ): Promise { const { data, encoding, error } = await messageEncoding.decodeData( - stateData.value, - stateData.encoding, + objectData.value, + objectData.encoding, inputContext, ); - stateData.value = data; - stateData.encoding = encoding ?? undefined; + objectData.value = data; + objectData.encoding = encoding ?? undefined; if (error) { throw error; } } - private static _encodeStateOperation(stateOperation: StateOperation, encodeFn: EncodeFunction): StateOperation { - // deep copy "stateOperation" object so we can modify the copy here. + private static _encodeObjectOperation(objectOperation: ObjectOperation, encodeFn: EncodeFunction): ObjectOperation { + // deep copy "objectOperation" object so we can modify the copy here. // buffer values won't be correctly copied, so we will need to set them again explicitly. - const stateOperationCopy = JSON.parse(JSON.stringify(stateOperation)) as StateOperation; + const objectOperationCopy = JSON.parse(JSON.stringify(objectOperation)) as ObjectOperation; - if (stateOperationCopy.mapOp?.data && 'value' in stateOperationCopy.mapOp.data) { - // use original "stateOperation" object when encoding values, so we have access to the original buffer values. - stateOperationCopy.mapOp.data = StateMessage._encodeStateData(stateOperation.mapOp?.data!, encodeFn); + if (objectOperationCopy.mapOp?.data && 'value' in objectOperationCopy.mapOp.data) { + // use original "objectOperation" object when encoding values, so we have access to the original buffer values. + objectOperationCopy.mapOp.data = ObjectMessage._encodeObjectData(objectOperation.mapOp?.data!, encodeFn); } - if (stateOperationCopy.map?.entries) { - Object.entries(stateOperationCopy.map.entries).forEach(([key, entry]) => { - // use original "stateOperation" object when encoding values, so we have access to original buffer values. - entry.data = StateMessage._encodeStateData(stateOperation?.map?.entries?.[key].data!, encodeFn); + if (objectOperationCopy.map?.entries) { + Object.entries(objectOperationCopy.map.entries).forEach(([key, entry]) => { + // use original "objectOperation" object when encoding values, so we have access to original buffer values. + entry.data = ObjectMessage._encodeObjectData(objectOperation?.map?.entries?.[key].data!, encodeFn); }); } - if (stateOperation.initialValue) { - // use original "stateOperation" object so we have access to the original buffer value - const { data: encodedInitialValue } = encodeFn(stateOperation.initialValue); - stateOperationCopy.initialValue = encodedInitialValue; + if (objectOperation.initialValue) { + // use original "objectOperation" object so we have access to the original buffer value + const { data: encodedInitialValue } = encodeFn(objectOperation.initialValue); + objectOperationCopy.initialValue = encodedInitialValue; } - return stateOperationCopy; + return objectOperationCopy; } - private static _encodeStateObject(stateObject: StateObject, encodeFn: EncodeFunction): StateObject { - // deep copy "stateObject" object so we can modify the copy here. + private static _encodeObjectState(objectState: ObjectState, encodeFn: EncodeFunction): ObjectState { + // deep copy "objectState" object so we can modify the copy here. // buffer values won't be correctly copied, so we will need to set them again explicitly. - const stateObjectCopy = JSON.parse(JSON.stringify(stateObject)) as StateObject; + const objectStateCopy = JSON.parse(JSON.stringify(objectState)) as ObjectState; - if (stateObjectCopy.map?.entries) { - Object.entries(stateObjectCopy.map.entries).forEach(([key, entry]) => { - // use original "stateObject" object when encoding values, so we have access to original buffer values. - entry.data = StateMessage._encodeStateData(stateObject?.map?.entries?.[key].data!, encodeFn); + if (objectStateCopy.map?.entries) { + Object.entries(objectStateCopy.map.entries).forEach(([key, entry]) => { + // use original "objectState" object when encoding values, so we have access to original buffer values. + entry.data = ObjectMessage._encodeObjectData(objectState?.map?.entries?.[key].data!, encodeFn); }); } - if (stateObjectCopy.createOp) { - // use original "stateObject" object when encoding values, so we have access to original buffer values. - stateObjectCopy.createOp = StateMessage._encodeStateOperation(stateObject.createOp!, encodeFn); + if (objectStateCopy.createOp) { + // use original "objectState" object when encoding values, so we have access to original buffer values. + objectStateCopy.createOp = ObjectMessage._encodeObjectOperation(objectState.createOp!, encodeFn); } - return stateObjectCopy; + return objectStateCopy; } - private static _encodeStateData(data: StateData, encodeFn: EncodeFunction): StateData { + private static _encodeObjectData(data: ObjectData, encodeFn: EncodeFunction): ObjectData { const { data: encodedValue, encoding: newEncoding } = encodeFn(data?.value, data?.encoding); return { @@ -361,17 +371,17 @@ export class StateMessage { } /** - * Encodes operation and object fields of the StateMessage. Does not mutate the provided StateMessage. + * Encodes operation and object fields of the ObjectMessage. Does not mutate the provided ObjectMessage. * * Uses encoding functions from regular `Message` processing. */ private static _encodeForWireProtocol( - message: StateMessage, + message: ObjectMessage, messageEncoding: typeof MessageEncoding, format: Utils.Format, ): { - operation?: StateOperation; - object?: StateObject; + operation?: ObjectOperation; + objectState?: ObjectState; } { const encodeFn: EncodeFunction = (data, encoding) => { const { data: encodedData, encoding: newEncoding } = messageEncoding.encodeDataForWireProtocol( @@ -386,13 +396,13 @@ export class StateMessage { }; const encodedOperation = message.operation - ? StateMessage._encodeStateOperation(message.operation, encodeFn) + ? ObjectMessage._encodeObjectOperation(message.operation, encodeFn) : undefined; - const encodedObject = message.object ? StateMessage._encodeStateObject(message.object, encodeFn) : undefined; + const encodedObjectState = message.object ? ObjectMessage._encodeObjectState(message.object, encodeFn) : undefined; return { operation: encodedOperation, - object: encodedObject, + objectState: encodedObjectState, }; } @@ -406,27 +416,27 @@ export class StateMessage { toJSON(): { id?: string; clientId?: string; - operation?: StateOperation; - object?: StateObject; + operation?: ObjectOperation; + object?: ObjectState; extras?: any; } { // we can infer the format used by client by inspecting with what arguments this method was called. // if JSON protocol is being used, the JSON.stringify() will be called and this toJSON() method will have a non-empty arguments list. // MSGPack protocol implementation also calls toJSON(), but with an empty arguments list. const format = arguments.length > 0 ? this._utils.Format.json : this._utils.Format.msgpack; - const { operation, object } = StateMessage._encodeForWireProtocol(this, this._messageEncoding, format); + const { operation, objectState } = ObjectMessage._encodeForWireProtocol(this, this._messageEncoding, format); return { id: this.id, clientId: this.clientId, operation, - object, + object: objectState, extras: this.extras, }; } toString(): string { - let result = '[StateMessage'; + let result = '[ObjectMessage'; if (this.id) result += '; id=' + this.id; if (this.timestamp) result += '; timestamp=' + this.timestamp; @@ -451,10 +461,10 @@ export class StateMessage { size += this.clientId?.length ?? 0; if (this.operation) { - size += this._getStateOperationSize(this.operation); + size += this._getObjectOperationSize(this.operation); } if (this.object) { - size += this._getStateObjectSize(this.object); + size += this._getObjectStateSize(this.object); } if (this.extras) { size += JSON.stringify(this.extras).length; @@ -463,55 +473,55 @@ export class StateMessage { return size; } - private _getStateOperationSize(operation: StateOperation): number { + private _getObjectOperationSize(operation: ObjectOperation): number { let size = 0; if (operation.mapOp) { - size += this._getStateMapOpSize(operation.mapOp); + size += this._getMapOpSize(operation.mapOp); } if (operation.counterOp) { - size += this._getStateCounterOpSize(operation.counterOp); + size += this._getCounterOpSize(operation.counterOp); } if (operation.map) { - size += this._getStateMapSize(operation.map); + size += this._getObjectMapSize(operation.map); } if (operation.counter) { - size += this._getStateCounterSize(operation.counter); + size += this._getObjectCounterSize(operation.counter); } return size; } - private _getStateObjectSize(obj: StateObject): number { + private _getObjectStateSize(obj: ObjectState): number { let size = 0; if (obj.map) { - size += this._getStateMapSize(obj.map); + size += this._getObjectMapSize(obj.map); } if (obj.counter) { - size += this._getStateCounterSize(obj.counter); + size += this._getObjectCounterSize(obj.counter); } if (obj.createOp) { - size += this._getStateOperationSize(obj.createOp); + size += this._getObjectOperationSize(obj.createOp); } return size; } - private _getStateMapSize(map: StateMap): number { + private _getObjectMapSize(map: ObjectMap): number { let size = 0; Object.entries(map.entries ?? {}).forEach(([key, entry]) => { size += key?.length ?? 0; if (entry) { - size += this._getStateMapEntrySize(entry); + size += this._getMapEntrySize(entry); } }); return size; } - private _getStateCounterSize(counter: StateCounter): number { + private _getObjectCounterSize(counter: ObjectCounter): number { if (counter.count == null) { return 0; } @@ -519,29 +529,29 @@ export class StateMessage { return 8; } - private _getStateMapEntrySize(entry: StateMapEntry): number { + private _getMapEntrySize(entry: MapEntry): number { let size = 0; if (entry.data) { - size += this._getStateDataSize(entry.data); + size += this._getObjectDataSize(entry.data); } return size; } - private _getStateMapOpSize(mapOp: StateMapOp): number { + private _getMapOpSize(mapOp: MapOp): number { let size = 0; size += mapOp.key?.length ?? 0; if (mapOp.data) { - size += this._getStateDataSize(mapOp.data); + size += this._getObjectDataSize(mapOp.data); } return size; } - private _getStateCounterOpSize(operation: StateCounterOp): number { + private _getCounterOpSize(operation: CounterOp): number { if (operation.amount == null) { return 0; } @@ -549,17 +559,17 @@ export class StateMessage { return 8; } - private _getStateDataSize(data: StateData): number { + private _getObjectDataSize(data: ObjectData): number { let size = 0; if (data.value) { - size += this._getStateValueSize(data.value); + size += this._getObjectValueSize(data.value); } return size; } - private _getStateValueSize(value: StateValue): number { + private _getObjectValueSize(value: ObjectValue): number { return this._utils.dataSizeBytes(value); } } diff --git a/src/plugins/objects/syncobjectsdatapool.ts b/src/plugins/objects/syncobjectsdatapool.ts index 2805efe2e2..478a7590fc 100644 --- a/src/plugins/objects/syncobjectsdatapool.ts +++ b/src/plugins/objects/syncobjectsdatapool.ts @@ -1,10 +1,10 @@ import type BaseClient from 'common/lib/client/baseclient'; import type RealtimeChannel from 'common/lib/client/realtimechannel'; import { Objects } from './objects'; -import { StateMessage, StateObject } from './statemessage'; +import { ObjectMessage, ObjectState } from './statemessage'; export interface LiveObjectDataEntry { - stateObject: StateObject; + objectState: ObjectState; objectType: 'LiveMap' | 'LiveCounter'; } @@ -49,47 +49,47 @@ export class SyncObjectsDataPool { this._pool.clear(); } - applyStateSyncMessages(stateMessages: StateMessage[]): void { - for (const stateMessage of stateMessages) { - if (!stateMessage.object) { + applyObjectSyncMessages(objectMessages: ObjectMessage[]): void { + for (const objectMessage of objectMessages) { + if (!objectMessage.object) { this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MAJOR, - 'SyncObjectsDataPool.applyStateSyncMessages()', - `state object message is received during SYNC without 'object' field, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + 'SyncObjectsDataPool.applyObjectSyncMessages()', + `object message is received during OBJECT_SYNC without 'object' field, skipping message; message id: ${objectMessage.id}, channel: ${this._channel.name}`, ); continue; } - const stateObject = stateMessage.object; + const objectState = objectMessage.object; - if (stateObject.counter) { - this._pool.set(stateObject.objectId, this._createLiveCounterDataEntry(stateObject)); - } else if (stateObject.map) { - this._pool.set(stateObject.objectId, this._createLiveMapDataEntry(stateObject)); + if (objectState.counter) { + this._pool.set(objectState.objectId, this._createLiveCounterDataEntry(objectState)); + } else if (objectState.map) { + this._pool.set(objectState.objectId, this._createLiveMapDataEntry(objectState)); } else { this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MAJOR, - 'SyncObjectsDataPool.applyStateSyncMessages()', - `received unsupported state object message during SYNC, expected 'counter' or 'map' to be present, skipping message; message id: ${stateMessage.id}, channel: ${this._channel.name}`, + 'SyncObjectsDataPool.applyObjectSyncMessages()', + `received unsupported object state message during OBJECT_SYNC, expected 'counter' or 'map' to be present, skipping message; message id: ${objectMessage.id}, channel: ${this._channel.name}`, ); } } } - private _createLiveCounterDataEntry(stateObject: StateObject): LiveCounterDataEntry { + private _createLiveCounterDataEntry(objectState: ObjectState): LiveCounterDataEntry { const newEntry: LiveCounterDataEntry = { - stateObject, + objectState, objectType: 'LiveCounter', }; return newEntry; } - private _createLiveMapDataEntry(stateObject: StateObject): LiveMapDataEntry { + private _createLiveMapDataEntry(objectState: ObjectState): LiveMapDataEntry { const newEntry: LiveMapDataEntry = { - stateObject, + objectState, objectType: 'LiveMap', }; diff --git a/test/common/modules/objects_helper.js b/test/common/modules/objects_helper.js index 2580803103..1d3dacf973 100644 --- a/test/common/modules/objects_helper.js +++ b/test/common/modules/objects_helper.js @@ -1,7 +1,7 @@ 'use strict'; /** - * Objects helper to create pre-determined state tree on channels + * Helper class to create pre-determined objects tree on channels and create object messages. */ define(['ably', 'shared_helper', 'objects'], function (Ably, Helper, ObjectsPlugin) { const createPM = Ably.makeProtocolMessageFromDeserialized({ ObjectsPlugin }); @@ -32,7 +32,7 @@ define(['ably', 'shared_helper', 'objects'], function (Ably, Helper, ObjectsPlug } /** - * Sends REST STATE requests to create Objects state tree on a provided channel name: + * Sends Objects REST API requests to create objects tree on a provided channel: * * root "emptyMap" -> Map#1 {} -- empty map * root "referencedMap" -> Map#2 { "counterKey": } @@ -240,41 +240,41 @@ define(['ably', 'shared_helper', 'objects'], function (Ably, Helper, ObjectsPlug return obj; } - stateOperationMessage(opts) { + objectOperationMessage(opts) { const { channelName, serial, siteCode, state } = opts; - state?.forEach((stateMessage, i) => { - stateMessage.serial = serial; - stateMessage.siteCode = siteCode; + state?.forEach((objectMessage, i) => { + objectMessage.serial = serial; + objectMessage.siteCode = siteCode; }); return { - action: 19, // STATE + action: 19, // OBJECT channel: channelName, channelSerial: serial, state: state ?? [], }; } - stateObjectMessage(opts) { + objectStateMessage(opts) { const { channelName, syncSerial, state } = opts; return { - action: 20, // STATE_SYNC + action: 20, // OBJECT_SYNC channel: channelName, channelSerial: syncSerial, state: state ?? [], }; } - async processStateOperationMessageOnChannel(opts) { + async processObjectOperationMessageOnChannel(opts) { const { channel, ...rest } = opts; this._helper.recordPrivateApi('call.channel.processMessage'); this._helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); await channel.processMessage( createPM( - this.stateOperationMessage({ + this.objectOperationMessage({ ...rest, channelName: channel.name, }), @@ -282,14 +282,14 @@ define(['ably', 'shared_helper', 'objects'], function (Ably, Helper, ObjectsPlug ); } - async processStateObjectMessageOnChannel(opts) { + async processObjectStateMessageOnChannel(opts) { const { channel, ...rest } = opts; this._helper.recordPrivateApi('call.channel.processMessage'); this._helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); await channel.processMessage( createPM( - this.stateObjectMessage({ + this.objectStateMessage({ ...rest, channelName: channel.name, }), diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index e199afa6a2..986e8d8dc7 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -22,13 +22,13 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'call.Objects._objectsPool.get', 'call.Message.decode', 'call.Message.encode', + 'call.ObjectMessage.encode', + 'call.ObjectMessage.fromValues', + 'call.ObjectMessage.getMessageSize', 'call.Platform.Config.push.storage.clear', 'call.Platform.nextTick', 'call.PresenceMessage.fromValues', 'call.ProtocolMessage.setFlag', - 'call.StateMessage.encode', - 'call.StateMessage.fromValues', - 'call.StateMessage.getMessageSize', 'call.Utils.copy', 'call.Utils.dataSizeBytes', 'call.Utils.getRetryTime', diff --git a/test/package/browser/template/src/index-objects.ts b/test/package/browser/template/src/index-objects.ts index fa8e0c1707..bed8e14216 100644 --- a/test/package/browser/template/src/index-objects.ts +++ b/test/package/browser/template/src/index-objects.ts @@ -31,7 +31,7 @@ globalThis.testAblyPackage = async function () { const aString: string | undefined = root.get('stringKey'); const aBoolean: boolean | undefined = root.get('booleanKey'); const userProvidedUndefined: string | undefined = root.get('couldBeUndefined'); - // live objects on a root: + // objects on a root: const counter: Ably.LiveCounter | undefined = root.get('counterKey'); const map: ObjectsTypes['root']['mapKey'] | undefined = root.get('mapKey'); // check string literal types works diff --git a/test/realtime/channel.test.js b/test/realtime/channel.test.js index a342b5a5dc..81c3faa05e 100644 --- a/test/realtime/channel.test.js +++ b/test/realtime/channel.test.js @@ -1934,9 +1934,9 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channelSerial: 'PRESENCE', }), createPM({ - action: 19, // STATE + action: 19, // OBJECT channel: channel.name, - channelSerial: 'STATE', + channelSerial: 'OBJECT', }), ]; diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js index 116ffd2ab7..fc892e44b6 100644 --- a/test/realtime/objects.test.js +++ b/test/realtime/objects.test.js @@ -87,8 +87,8 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function return savedError; } - function stateMessageFromValues(values) { - return ObjectsPlugin.StateMessage.fromValues(values, Utils, MessageEncoding); + function objectMessageFromValues(values) { + return ObjectsPlugin.ObjectMessage.fromValues(values, Utils, MessageEncoding); } async function waitForMapKeyUpdate(map, key) { @@ -111,7 +111,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); } - async function waitForStateOperation(helper, client, waitForAction) { + async function waitForObjectOperation(helper, client, waitForAction) { return new Promise((resolve, reject) => { helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); const transport = client.connection.connectionManager.activeProtocol.getTransport(); @@ -136,7 +136,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function } /** - * The channel with fixture data may not yet be populated by REST STATE requests made by ObjectsHelper. + * The channel with fixture data may not yet be populated by REST API requests made by ObjectsHelper. * This function waits for a channel to have all keys set. */ async function waitFixtureChannelIsReady(client) { @@ -179,7 +179,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); /** @nospec */ - it(`doesn't break when it receives a STATE ProtocolMessage`, async function () { + it(`doesn't break when it receives an OBJECT ProtocolMessage`, async function () { const helper = this.test.helper; const objectsHelper = new ObjectsHelper(helper); const testClient = helper.AblyRealtime(); @@ -193,8 +193,8 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const publishClient = helper.AblyRealtime(); await helper.monitorConnectionThenCloseAndFinish(async () => { - // inject STATE message that should be ignored and not break anything without the plugin - await objectsHelper.processStateOperationMessageOnChannel({ + // inject OBJECT message that should be ignored and not break anything without the plugin + await objectsHelper.processObjectOperationMessageOnChannel({ channel: testChannel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', @@ -204,14 +204,14 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const publishChannel = publishClient.channels.get('channel'); await publishChannel.publish(null, 'test'); - // regular message subscriptions should still work after processing STATE_SYNC message without the plugin + // regular message subscriptions should still work after processing OBJECT_SYNC message without the plugin await receivedMessagePromise; }, publishClient); }, testClient); }); /** @nospec */ - it(`doesn't break when it receives a STATE_SYNC ProtocolMessage`, async function () { + it(`doesn't break when it receives an OBJECT_SYNC ProtocolMessage`, async function () { const helper = this.test.helper; const objectsHelper = new ObjectsHelper(helper); const testClient = helper.AblyRealtime(); @@ -225,8 +225,8 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const publishClient = helper.AblyRealtime(); await helper.monitorConnectionThenCloseAndFinish(async () => { - // inject STATE_SYNC message that should be ignored and not break anything without the plugin - await objectsHelper.processStateObjectMessageOnChannel({ + // inject OBJECT_SYNC message that should be ignored and not break anything without the plugin + await objectsHelper.processObjectStateMessageOnChannel({ channel: testChannel, syncSerial: 'serial:', state: [ @@ -240,7 +240,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const publishChannel = publishClient.channels.get('channel'); await publishChannel.publish(null, 'test'); - // regular message subscriptions should still work after processing STATE_SYNC message without the plugin + // regular message subscriptions should still work after processing OBJECT_SYNC message without the plugin await receivedMessagePromise; }, publishClient); }, testClient); @@ -273,7 +273,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); /** @nospec */ - it('getRoot() returns live object with id "root"', async function () { + it('getRoot() returns LiveObject with id "root"', async function () { const helper = this.test.helper; const client = RealtimeWithObjects(helper); @@ -290,7 +290,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); /** @nospec */ - it('getRoot() returns empty root when no state exist on a channel', async function () { + it('getRoot() returns empty root when no objects exist on a channel', async function () { const helper = this.test.helper; const client = RealtimeWithObjects(helper); @@ -306,7 +306,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); /** @nospec */ - it('getRoot() waits for initial STATE_SYNC to be completed before resolving', async function () { + it('getRoot() waits for initial OBJECT_SYNC to be completed before resolving', async function () { const helper = this.test.helper; const client = RealtimeWithObjects(helper); @@ -324,7 +324,8 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // give a chance for getRoot() to resolve and proc its handler. it should not helper.recordPrivateApi('call.Platform.nextTick'); await new Promise((res) => nextTick(res)); - expect(getRootResolved, 'Check getRoot() is not resolved until STATE_SYNC sequence is completed').to.be.false; + expect(getRootResolved, 'Check getRoot() is not resolved until OBJECT_SYNC sequence is completed').to.be + .false; await channel.attach(); @@ -334,7 +335,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); /** @nospec */ - it('getRoot() resolves immediately when STATE_SYNC sequence is completed', async function () { + it('getRoot() resolves immediately when OBJECT_SYNC sequence is completed', async function () { const helper = this.test.helper; const client = RealtimeWithObjects(helper); @@ -343,7 +344,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const objects = channel.objects; await channel.attach(); - // wait for STATE_SYNC sequence to complete by accessing root for the first time + // wait for sync sequence to complete by accessing root for the first time await objects.getRoot(); let resolvedImmediately = false; @@ -360,7 +361,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); /** @nospec */ - it('getRoot() waits for STATE_SYNC with empty cursor before resolving', async function () { + it('getRoot() waits for OBJECT_SYNC with empty cursor before resolving', async function () { const helper = this.test.helper; const objectsHelper = new ObjectsHelper(helper); const client = RealtimeWithObjects(helper); @@ -370,13 +371,13 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const objects = channel.objects; await channel.attach(); - // wait for initial STATE_SYNC sequence to complete + // wait for initial sync sequence to complete await objects.getRoot(); - // inject STATE_SYNC message to emulate start of a new sequence - await objectsHelper.processStateObjectMessageOnChannel({ + // inject OBJECT_SYNC message to emulate start of a new sequence + await objectsHelper.processObjectStateMessageOnChannel({ channel, - // have cursor so client awaits for additional STATE_SYNC messages + // have cursor so client awaits for additional OBJECT_SYNC messages syncSerial: 'serial:cursor', }); @@ -391,12 +392,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function helper.recordPrivateApi('call.Platform.nextTick'); await new Promise((res) => nextTick(res)); - expect(getRootResolved, 'Check getRoot() is not resolved while STATE_SYNC is in progress').to.be.false; + expect(getRootResolved, 'Check getRoot() is not resolved while OBJECT_SYNC is in progress').to.be.false; - // inject final STATE_SYNC message - await objectsHelper.processStateObjectMessageOnChannel({ + // inject final OBJECT_SYNC message + await objectsHelper.processObjectStateMessageOnChannel({ channel, - // no cursor to indicate the end of STATE_SYNC messages + // no cursor to indicate the end of OBJECT_SYNC messages syncSerial: 'serial:', state: [ objectsHelper.mapObject({ @@ -411,15 +412,15 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function helper.recordPrivateApi('call.Platform.nextTick'); await new Promise((res) => nextTick(res)); - expect(getRootResolved, 'Check getRoot() is resolved when STATE_SYNC sequence has ended').to.be.true; - expect(root.get('key')).to.equal(1, 'Check new root after STATE_SYNC sequence has expected key'); + expect(getRootResolved, 'Check getRoot() is resolved when OBJECT_SYNC sequence has ended').to.be.true; + expect(root.get('key')).to.equal(1, 'Check new root after OBJECT_SYNC sequence has expected key'); }, client); }); /** @nospec */ Helper.testOnAllTransportsAndProtocols( this, - 'builds state object tree from STATE_SYNC sequence on channel attachment', + 'builds object tree from OBJECT_SYNC sequence on channel attachment', function (options, channelName) { return async function () { const helper = this.test.helper; @@ -438,7 +439,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const mapKeys = ['emptyMap', 'referencedMap', 'valuesMap']; const rootKeysCount = counterKeys.length + mapKeys.length; - expect(root, 'Check getRoot() is resolved when STATE_SYNC sequence ends').to.exist; + expect(root, 'Check getRoot() is resolved when OBJECT_SYNC sequence ends').to.exist; expect(root.size()).to.equal(rootKeysCount, 'Check root has correct number of keys'); counterKeys.forEach((key) => { @@ -482,7 +483,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function /** @nospec */ Helper.testOnAllTransportsAndProtocols( this, - 'LiveCounter is initialized with initial value from STATE_SYNC sequence', + 'LiveCounter is initialized with initial value from OBJECT_SYNC sequence', function (options, channelName) { return async function () { const helper = this.test.helper; @@ -515,7 +516,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function /** @nospec */ Helper.testOnAllTransportsAndProtocols( this, - 'LiveMap is initialized with initial value from STATE_SYNC sequence', + 'LiveMap is initialized with initial value from OBJECT_SYNC sequence', function (options, channelName) { return async function () { const helper = this.test.helper; @@ -655,18 +656,18 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { name: 'negativeMaxSafeIntegerCounter', count: -Number.MAX_SAFE_INTEGER }, ]; - const stateSyncSequenceScenarios = [ + const objectSyncSequenceScenarios = [ { - description: 'STATE_SYNC sequence with state object "tombstone" property creates tombstoned object', + description: 'OBJECT_SYNC sequence with object state "tombstone" property creates tombstoned object', action: async (ctx) => { const { root, objectsHelper, channel } = ctx; const mapId = objectsHelper.fakeMapObjectId(); const counterId = objectsHelper.fakeCounterObjectId(); - await objectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processObjectStateMessageOnChannel({ channel, - syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately - // add state objects with tombstone=true + syncSerial: 'serial:', // empty serial so sync sequence ends immediately + // add object states with tombstone=true state: [ objectsHelper.mapObject({ objectId: mapId, @@ -698,20 +699,20 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function expect( root.get('map'), - 'Check map does not exist on root after STATE_SYNC with "tombstone=true" for a map object', + 'Check map does not exist on root after OBJECT_SYNC with "tombstone=true" for a map object', ).to.not.exist; expect( root.get('counter'), - 'Check counter does not exist on root after STATE_SYNC with "tombstone=true" for a counter object', + 'Check counter does not exist on root after OBJECT_SYNC with "tombstone=true" for a counter object', ).to.not.exist; - // control check that STATE_SYNC was applied at all - expect(root.get('foo'), 'Check property exists on root after STATE_SYNC').to.exist; + // control check that OBJECT_SYNC was applied at all + expect(root.get('foo'), 'Check property exists on root after OBJECT_SYNC').to.exist; }, }, { allTransportsAndProtocols: true, - description: 'STATE_SYNC sequence with state object "tombstone" property deletes existing object', + description: 'OBJECT_SYNC sequence with object state "tombstone" property deletes existing object', action: async (ctx) => { const { root, objectsHelper, channelName, channel } = ctx; @@ -723,13 +724,15 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); await counterCreatedPromise; - expect(root.get('counter'), 'Check counter exists on root before STATE_SYNC sequence with "tombstone=true"') - .to.exist; + expect( + root.get('counter'), + 'Check counter exists on root before OBJECT_SYNC sequence with "tombstone=true"', + ).to.exist; - // inject a STATE_SYNC sequence where a counter is now tombstoned - await objectsHelper.processStateObjectMessageOnChannel({ + // inject an OBJECT_SYNC message where a counter is now tombstoned + await objectsHelper.processObjectStateMessageOnChannel({ channel, - syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately + syncSerial: 'serial:', // empty serial so sync sequence ends immediately state: [ objectsHelper.counterObject({ objectId: counterId, @@ -752,17 +755,17 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function expect( root.get('counter'), - 'Check counter does not exist on root after STATE_SYNC with "tombstone=true" for an existing counter object', + 'Check counter does not exist on root after OBJECT_SYNC with "tombstone=true" for an existing counter object', ).to.not.exist; - // control check that STATE_SYNC was applied at all - expect(root.get('foo'), 'Check property exists on root after STATE_SYNC').to.exist; + // control check that OBJECT_SYNC was applied at all + expect(root.get('foo'), 'Check property exists on root after OBJECT_SYNC').to.exist; }, }, { allTransportsAndProtocols: true, description: - 'STATE_SYNC sequence with state object "tombstone" property triggers subscription callback for existing object', + 'OBJECT_SYNC sequence with object state "tombstone" property triggers subscription callback for existing object', action: async (ctx) => { const { root, objectsHelper, channelName, channel } = ctx; @@ -779,7 +782,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function try { expect(update).to.deep.equal( { update: { amount: -1 } }, - 'Check counter subscription callback is called with an expected update object after STATE_SYNC sequence with "tombstone=true"', + 'Check counter subscription callback is called with an expected update object after OBJECT_SYNC sequence with "tombstone=true"', ); resolve(); } catch (error) { @@ -788,10 +791,10 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }), ); - // inject a STATE_SYNC sequence where a counter is now tombstoned - await objectsHelper.processStateObjectMessageOnChannel({ + // inject an OBJECT_SYNC message where a counter is now tombstoned + await objectsHelper.processObjectStateMessageOnChannel({ channel, - syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately + syncSerial: 'serial:', // empty serial so sync sequence ends immediately state: [ objectsHelper.counterObject({ objectId: counterId, @@ -819,7 +822,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const applyOperationsScenarios = [ { allTransportsAndProtocols: true, - description: 'can apply MAP_CREATE with primitives state operation messages', + description: 'can apply MAP_CREATE with primitives object operation messages', action: async (ctx) => { const { root, objectsHelper, channelName, helper } = ctx; @@ -883,7 +886,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { allTransportsAndProtocols: true, - description: 'can apply MAP_CREATE with object ids state operation messages', + description: 'can apply MAP_CREATE with object ids object operation messages', action: async (ctx) => { const { root, objectsHelper, channelName } = ctx; const withReferencesMapKey = 'withReferencesMap'; @@ -959,7 +962,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { description: - 'MAP_CREATE state operation messages are applied based on the site timeserials vector of the object', + 'MAP_CREATE object operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { const { root, objectsHelper, channel } = ctx; @@ -974,13 +977,13 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await Promise.all( mapIds.map(async (mapId, i) => { // send a MAP_SET op first to create a zero-value map with forged site timeserials vector (from the op), and set it on a root. - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', state: [objectsHelper.mapSetOp({ objectId: mapId, key: 'foo', data: { value: 'bar' } })], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', @@ -997,7 +1000,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied ].entries()) { - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial, siteCode, @@ -1041,7 +1044,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { allTransportsAndProtocols: true, - description: 'can apply MAP_SET with primitives state operation messages', + description: 'can apply MAP_SET with primitives object operation messages', action: async (ctx) => { const { root, objectsHelper, channelName, helper } = ctx; @@ -1090,7 +1093,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { allTransportsAndProtocols: true, - description: 'can apply MAP_SET with object ids state operation messages', + description: 'can apply MAP_SET with object ids object operation messages', action: async (ctx) => { const { root, objectsHelper, channelName } = ctx; @@ -1148,13 +1151,13 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { description: - 'MAP_SET state operation messages are applied based on the site timeserials vector of the object', + 'MAP_SET object operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { const { root, objectsHelper, channel } = ctx; // create new map and set it on a root with forged timeserials const mapId = objectsHelper.fakeMapObjectId(); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', @@ -1172,7 +1175,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }), ], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', @@ -1188,7 +1191,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier entry CGO, not applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later entry CGO, applied ].entries()) { - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial, siteCode, @@ -1217,7 +1220,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { allTransportsAndProtocols: true, - description: 'can apply MAP_REMOVE state operation messages', + description: 'can apply MAP_REMOVE object operation messages', action: async (ctx) => { const { root, objectsHelper, channelName } = ctx; const mapKey = 'map'; @@ -1280,13 +1283,13 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { description: - 'MAP_REMOVE state operation messages are applied based on the site timeserials vector of the object', + 'MAP_REMOVE object operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { const { root, objectsHelper, channel } = ctx; // create new map and set it on a root with forged timeserials const mapId = objectsHelper.fakeMapObjectId(); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', @@ -1304,7 +1307,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }), ], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', @@ -1320,7 +1323,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier entry CGO, not applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later entry CGO, applied ].entries()) { - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial, siteCode, @@ -1352,7 +1355,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { allTransportsAndProtocols: true, - description: 'can apply COUNTER_CREATE state operation messages', + description: 'can apply COUNTER_CREATE object operation messages', action: async (ctx) => { const { root, objectsHelper, channelName } = ctx; @@ -1405,7 +1408,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { description: - 'COUNTER_CREATE state operation messages are applied based on the site timeserials vector of the object', + 'COUNTER_CREATE object operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { const { root, objectsHelper, channel } = ctx; @@ -1420,13 +1423,13 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await Promise.all( counterIds.map(async (counterId, i) => { // send a COUNTER_INC op first to create a zero-value counter with forged site timeserials vector (from the op), and set it on a root. - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', state: [objectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', @@ -1443,7 +1446,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied ].entries()) { - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial, siteCode, @@ -1473,7 +1476,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { allTransportsAndProtocols: true, - description: 'can apply COUNTER_INC state operation messages', + description: 'can apply COUNTER_INC object operation messages', action: async (ctx) => { const { root, objectsHelper, channelName } = ctx; const counterKey = 'counter'; @@ -1537,19 +1540,19 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { description: - 'COUNTER_INC state operation messages are applied based on the site timeserials vector of the object', + 'COUNTER_INC object operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { const { root, objectsHelper, channel } = ctx; // create new counter and set it on a root with forged timeserials const counterId = objectsHelper.fakeCounterObjectId(); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', state: [objectsHelper.counterCreateOp({ objectId: counterId, count: 1 })], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', @@ -1565,7 +1568,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // +100000 different site, earlier CGO, applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // +1000000 different site, later CGO, applied ].entries()) { - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial, siteCode, @@ -1582,7 +1585,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { - description: 'can apply OBJECT_DELETE state operation messages', + description: 'can apply OBJECT_DELETE object operation messages', action: async (ctx) => { const { root, objectsHelper, channelName, channel } = ctx; @@ -1607,13 +1610,13 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function expect(root.get('counter'), 'Check counter exists on root before OBJECT_DELETE').to.exist; // inject OBJECT_DELETE - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', state: [objectsHelper.objectDeleteOp({ objectId: mapObjectId })], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 0), siteCode: 'aaa', @@ -1632,7 +1635,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const counterId = objectsHelper.fakeCounterObjectId(); // inject OBJECT_DELETE. should create a zero-value tombstoned object which can't be modified - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', @@ -1640,13 +1643,13 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); // try to create and set tombstoned object on root - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb', state: [objectsHelper.counterCreateOp({ objectId: counterId })], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', @@ -1659,7 +1662,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { description: - 'OBJECT_DELETE state operation messages are applied based on the site timeserials vector of the object', + 'OBJECT_DELETE object operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { const { root, objectsHelper, channel } = ctx; @@ -1674,13 +1677,13 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await Promise.all( counterIds.map(async (counterId, i) => { // create objects and set them on root with forged timeserials - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', state: [objectsHelper.counterCreateOp({ objectId: counterId })], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', @@ -1697,7 +1700,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied ].entries()) { - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial, siteCode, @@ -1787,13 +1790,13 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ); // inject OBJECT_DELETE - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', state: [objectsHelper.objectDeleteOp({ objectId: mapObjectId })], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 0), siteCode: 'aaa', @@ -1821,7 +1824,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function expect(root.get('foo'), 'Check counter exists on root before OBJECT_DELETE').to.exist; // inject OBJECT_DELETE - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', @@ -1829,7 +1832,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); // set tombstoned counter to another key on root - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', @@ -1842,7 +1845,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { - description: 'state operation message on a tombstoned object does not revive it', + description: 'object operation message on a tombstoned object does not revive it', action: async (ctx) => { const { root, objectsHelper, channelName, channel } = ctx; @@ -1874,39 +1877,39 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function expect(root.get('counter1'), 'Check counter1 exists on root before OBJECT_DELETE').to.exist; // inject OBJECT_DELETE - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', state: [objectsHelper.objectDeleteOp({ objectId: mapId1 })], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 0), siteCode: 'aaa', state: [objectsHelper.objectDeleteOp({ objectId: mapId2 })], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 2, 0), siteCode: 'aaa', state: [objectsHelper.objectDeleteOp({ objectId: counterId1 })], }); - // inject state ops on tombstoned objects - await objectsHelper.processStateOperationMessageOnChannel({ + // inject object operations on tombstoned objects + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 3, 0), siteCode: 'aaa', state: [objectsHelper.mapSetOp({ objectId: mapId1, key: 'baz', data: { value: 'qux' } })], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 4, 0), siteCode: 'aaa', state: [objectsHelper.mapRemoveOp({ objectId: mapId2, key: 'foo' })], }); - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 5, 0), siteCode: 'aaa', @@ -1914,13 +1917,13 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); // objects should still be deleted - expect(root.get('map1'), 'Check map1 does not exist on root after OBJECT_DELETE and another state op').to + expect(root.get('map1'), 'Check map1 does not exist on root after OBJECT_DELETE and another object op').to .not.exist; - expect(root.get('map2'), 'Check map2 does not exist on root after OBJECT_DELETE and another state op').to + expect(root.get('map2'), 'Check map2 does not exist on root after OBJECT_DELETE and another object op').to .not.exist; expect( root.get('counter1'), - 'Check counter1 does not exist on root after OBJECT_DELETE and another state op', + 'Check counter1 does not exist on root after OBJECT_DELETE and another object op', ).to.not.exist; }, }, @@ -1928,12 +1931,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const applyOperationsDuringSyncScenarios = [ { - description: 'state operation messages are buffered during STATE_SYNC sequence', + description: 'object operation messages are buffered during OBJECT_SYNC sequence', action: async (ctx) => { const { root, objectsHelper, channel } = ctx; - // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages - await objectsHelper.processStateObjectMessageOnChannel({ + // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages + await objectsHelper.processObjectStateMessageOnChannel({ channel, syncSerial: 'serial:cursor', }); @@ -1941,7 +1944,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // inject operations, it should not be applied as sync is in progress await Promise.all( primitiveKeyData.map((keyData) => - objectsHelper.processStateOperationMessageOnChannel({ + objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', @@ -1953,19 +1956,19 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // check root doesn't have data from operations primitiveKeyData.forEach((keyData) => { - expect(root.get(keyData.key), `Check "${keyData.key}" key doesn't exist on root during STATE_SYNC`).to.not - .exist; + expect(root.get(keyData.key), `Check "${keyData.key}" key doesn't exist on root during OBJECT_SYNC`).to + .not.exist; }); }, }, { - description: 'buffered state operation messages are applied when STATE_SYNC sequence ends', + description: 'buffered object operation messages are applied when OBJECT_SYNC sequence ends', action: async (ctx) => { const { root, objectsHelper, channel, helper } = ctx; - // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages - await objectsHelper.processStateObjectMessageOnChannel({ + // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages + await objectsHelper.processObjectStateMessageOnChannel({ channel, syncSerial: 'serial:cursor', }); @@ -1973,7 +1976,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // inject operations, they should be applied when sync ends await Promise.all( primitiveKeyData.map((keyData, i) => - objectsHelper.processStateOperationMessageOnChannel({ + objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', @@ -1984,7 +1987,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ); // end the sync with empty cursor - await objectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processObjectStateMessageOnChannel({ channel, syncSerial: 'serial:', }); @@ -1996,12 +1999,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), - `Check root has correct value for "${keyData.key}" key after STATE_SYNC has ended and buffered operations are applied`, + `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, ).to.be.true; } else { expect(root.get(keyData.key)).to.equal( keyData.data.value, - `Check root has correct value for "${keyData.key}" key after STATE_SYNC has ended and buffered operations are applied`, + `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, ); } }); @@ -2009,12 +2012,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { - description: 'buffered state operation messages are discarded when new STATE_SYNC sequence starts', + description: 'buffered object operation messages are discarded when new OBJECT_SYNC sequence starts', action: async (ctx) => { const { root, objectsHelper, channel } = ctx; - // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages - await objectsHelper.processStateObjectMessageOnChannel({ + // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages + await objectsHelper.processObjectStateMessageOnChannel({ channel, syncSerial: 'serial:cursor', }); @@ -2022,7 +2025,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // inject operations, expect them to be discarded when sync with new sequence id starts await Promise.all( primitiveKeyData.map((keyData, i) => - objectsHelper.processStateOperationMessageOnChannel({ + objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', @@ -2033,13 +2036,13 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ); // start new sync with new sequence id - await objectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processObjectStateMessageOnChannel({ channel, syncSerial: 'otherserial:cursor', }); // inject another operation that should be applied when latest sync ends - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb', @@ -2047,7 +2050,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); // end sync - await objectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processObjectStateMessageOnChannel({ channel, syncSerial: 'otherserial:', }); @@ -2056,31 +2059,31 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function primitiveKeyData.forEach((keyData) => { expect( root.get(keyData.key), - `Check "${keyData.key}" key doesn't exist on root when STATE_SYNC has ended`, + `Check "${keyData.key}" key doesn't exist on root when OBJECT_SYNC has ended`, ).to.not.exist; }); // check root has data from operations received during second sync expect(root.get('foo')).to.equal( 'bar', - 'Check root has data from operations received during second STATE_SYNC sequence', + 'Check root has data from operations received during second OBJECT_SYNC sequence', ); }, }, { description: - 'buffered state operation messages are applied based on the site timeserials vector of the object', + 'buffered object operation messages are applied based on the site timeserials vector of the object', action: async (ctx) => { const { root, objectsHelper, channel } = ctx; - // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages + // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages const mapId = objectsHelper.fakeMapObjectId(); const counterId = objectsHelper.fakeCounterObjectId(); - await objectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processObjectStateMessageOnChannel({ channel, syncSerial: 'serial:cursor', - // add state object messages with non-empty site timeserials + // add object state messages with non-empty site timeserials state: [ // next map and counter objects will be checked to have correct operations applied on them based on site timeserials objectsHelper.mapObject({ @@ -2107,7 +2110,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, initialCount: 1, }), - // add objects to the root so they're discoverable in the state tree + // add objects to the root so they're discoverable in the object tree objectsHelper.mapObject({ objectId: 'root', siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, @@ -2133,7 +2136,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // different site with matching entry CGO case is not possible, as matching entry timeserial means that that timeserial is in the site timeserials vector { serial: lexicoTimeserial('ddd', 1, 0), siteCode: 'ddd' }, // different site, later entry CGO, applied ].entries()) { - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial, siteCode, @@ -2150,7 +2153,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // +100000 different site, earlier CGO, applied { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // +1000000 different site, later CGO, applied ].entries()) { - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial, siteCode, @@ -2159,7 +2162,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function } // end sync - await objectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processObjectStateMessageOnChannel({ channel, syncSerial: 'serial:', }); @@ -2179,25 +2182,25 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function expectedMapKeys.forEach(({ key, value }) => { expect(root.get('map').get(key)).to.equal( value, - `Check "${key}" key on map has expected value after STATE_SYNC has ended`, + `Check "${key}" key on map has expected value after OBJECT_SYNC has ended`, ); }); expect(root.get('counter').value()).to.equal( 1 + 1000 + 100000 + 1000000, // sum of passing operations and the initial value - `Check counter has expected value after STATE_SYNC has ended`, + `Check counter has expected value after OBJECT_SYNC has ended`, ); }, }, { description: - 'subsequent state operation messages are applied immediately after STATE_SYNC ended and buffers are applied', + 'subsequent object operation messages are applied immediately after OBJECT_SYNC ended and buffers are applied', action: async (ctx) => { const { root, objectsHelper, channel, channelName, helper } = ctx; - // start new sync sequence with a cursor so client will wait for the next STATE_SYNC messages - await objectsHelper.processStateObjectMessageOnChannel({ + // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages + await objectsHelper.processObjectStateMessageOnChannel({ channel, syncSerial: 'serial:cursor', }); @@ -2205,7 +2208,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // inject operations, they should be applied when sync ends await Promise.all( primitiveKeyData.map((keyData, i) => - objectsHelper.processStateOperationMessageOnChannel({ + objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', @@ -2216,7 +2219,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ); // end the sync with empty cursor - await objectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processObjectStateMessageOnChannel({ channel, syncSerial: 'serial:', }); @@ -2233,25 +2236,25 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ); await keyUpdatedPromise; - // check buffered operations are applied, as well as the most recent operation outside of the STATE_SYNC is applied + // check buffered operations are applied, as well as the most recent operation outside of the sync sequence is applied primitiveKeyData.forEach((keyData) => { if (keyData.data.encoding) { helper.recordPrivateApi('call.BufferUtils.base64Decode'); helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), - `Check root has correct value for "${keyData.key}" key after STATE_SYNC has ended and buffered operations are applied`, + `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, ).to.be.true; } else { expect(root.get(keyData.key)).to.equal( keyData.data.value, - `Check root has correct value for "${keyData.key}" key after STATE_SYNC has ended and buffered operations are applied`, + `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, ); } }); expect(root.get('foo')).to.equal( 'bar', - 'Check root has correct value for "foo" key from operation received outside of STATE_SYNC after other buffered operations were applied', + 'Check root has correct value for "foo" key from operation received outside of OBJECT_SYNC after other buffered operations were applied', ); }, }, @@ -2678,7 +2681,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { allTransportsAndProtocols: true, - description: 'LiveCounter created with Objects.createCounter can be assigned to the state tree', + description: 'LiveCounter created with Objects.createCounter can be assigned to the object tree', action: async (ctx) => { const { root, objects } = ctx; @@ -2699,7 +2702,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ); expect(root.get('counter').value()).to.equal( 1, - 'Check counter assigned to the state tree has the expected value', + 'Check counter assigned to the object tree has the expected value', ); }, }, @@ -2728,10 +2731,10 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // instead of sending CREATE op to the realtime, echo it immediately to the client // with forged initial value so we can check that counter gets initialized with a value from a CREATE op helper.recordPrivateApi('replace.Objects.publish'); - objects.publish = async (stateMessages) => { - const counterId = stateMessages[0].operation.objectId; + objects.publish = async (objectMessages) => { + const counterId = objectMessages[0].operation.objectId; // this should result execute regular operation application procedure and create an object in the pool with forged initial value - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 1), siteCode: 'aaa', @@ -2765,7 +2768,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const counterId = counter.getObjectId(); // now inject CREATE op for a counter with a forged value. it should not be applied - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 1), siteCode: 'aaa', @@ -2913,7 +2916,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { allTransportsAndProtocols: true, - description: 'LiveMap created with Objects.createMap can be assigned to the state tree', + description: 'LiveMap created with Objects.createMap can be assigned to the object tree', action: async (ctx) => { const { root, objects } = ctx; @@ -2928,15 +2931,15 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function expect(root.get('map')).to.equal(map, 'Check map object on root is the same as from create method'); expect(root.get('map').size()).to.equal( 2, - 'Check map assigned to the state tree has the expected number of keys', + 'Check map assigned to the object tree has the expected number of keys', ); expect(root.get('map').get('foo')).to.equal( 'bar', - 'Check map assigned to the state tree has the expected value for its string key', + 'Check map assigned to the object tree has the expected value for its string key', ); expect(root.get('map').get('baz')).to.equal( counter, - 'Check map assigned to the state tree has the expected value for its LiveCounter key', + 'Check map assigned to the object tree has the expected value for its LiveCounter key', ); }, }, @@ -2964,10 +2967,10 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // instead of sending CREATE op to the realtime, echo it immediately to the client // with forged initial value so we can check that map gets initialized with a value from a CREATE op helper.recordPrivateApi('replace.Objects.publish'); - objects.publish = async (stateMessages) => { - const mapId = stateMessages[0].operation.objectId; + objects.publish = async (objectMessages) => { + const mapId = objectMessages[0].operation.objectId; // this should result execute regular operation application procedure and create an object in the pool with forged initial value - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 1), siteCode: 'aaa', @@ -3007,7 +3010,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const mapId = map.getObjectId(); // now inject CREATE op for a map with a forged value. it should not be applied - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 1, 1), siteCode: 'aaa', @@ -3036,17 +3039,17 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function action: async (ctx) => { const { root, objects } = ctx; - await expectToThrowAsync(async () => objects.createMap(null), 'Map entries should be a key/value object'); - await expectToThrowAsync(async () => objects.createMap('foo'), 'Map entries should be a key/value object'); - await expectToThrowAsync(async () => objects.createMap(1), 'Map entries should be a key/value object'); + await expectToThrowAsync(async () => objects.createMap(null), 'Map entries should be a key-value object'); + await expectToThrowAsync(async () => objects.createMap('foo'), 'Map entries should be a key-value object'); + await expectToThrowAsync(async () => objects.createMap(1), 'Map entries should be a key-value object'); await expectToThrowAsync( async () => objects.createMap(BigInt(1)), - 'Map entries should be a key/value object', + 'Map entries should be a key-value object', ); - await expectToThrowAsync(async () => objects.createMap(true), 'Map entries should be a key/value object'); + await expectToThrowAsync(async () => objects.createMap(true), 'Map entries should be a key-value object'); await expectToThrowAsync( async () => objects.createMap(Symbol()), - 'Map entries should be a key/value object', + 'Map entries should be a key-value object', ); await expectToThrowAsync( @@ -3084,7 +3087,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { - description: 'batch API .get method on a map returns BatchContext* wrappers for live objects', + description: 'batch API .get method on a map returns BatchContext* wrappers for objects', action: async (ctx) => { const { root, objects } = ctx; @@ -3127,7 +3130,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { - description: 'batch API access API methods on live objects work and are synchronous', + description: 'batch API access API methods on objects work and are synchronous', action: async (ctx) => { const { root, objects } = ctx; @@ -3169,7 +3172,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { - description: 'batch API write API methods on live objects do not mutate objects inside the batch callback', + description: 'batch API write API methods on objects do not mutate objects inside the batch callback', action: async (ctx) => { const { root, objects } = ctx; @@ -3411,9 +3414,9 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const counterId1 = objectsHelper.fakeCounterObjectId(); const counterId2 = objectsHelper.fakeCounterObjectId(); - await objectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processObjectStateMessageOnChannel({ channel, - syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately + syncSerial: 'serial:', // empty serial so sync sequence ends immediately state: [ objectsHelper.counterObject({ objectId: counterId1, @@ -3469,9 +3472,9 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const counterId1 = objectsHelper.fakeCounterObjectId(); const counterId2 = objectsHelper.fakeCounterObjectId(); - await objectsHelper.processStateObjectMessageOnChannel({ + await objectsHelper.processObjectStateMessageOnChannel({ channel, - syncSerial: 'serial:', // empty serial so STATE_SYNC ends immediately + syncSerial: 'serial:', // empty serial so sync sequence ends immediately state: [ objectsHelper.counterObject({ objectId: counterId1, @@ -3533,7 +3536,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function forScenarios( this, [ - ...stateSyncSequenceScenarios, + ...objectSyncSequenceScenarios, ...applyOperationsScenarios, ...applyOperationsDuringSyncScenarios, ...writeApiScenarios, @@ -4092,7 +4095,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function action: async (ctx) => { const { objectsHelper, channelName, channel, objects, helper, waitForGCCycles, client } = ctx; - const counterCreatedPromise = waitForStateOperation(helper, client, ObjectsHelper.ACTIONS.COUNTER_CREATE); + const counterCreatedPromise = waitForObjectOperation(helper, client, ObjectsHelper.ACTIONS.COUNTER_CREATE); // send a CREATE op, this adds an object to the pool const { objectId } = await objectsHelper.stateRequest( channelName, @@ -4104,7 +4107,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function expect(objects._objectsPool.get(objectId), 'Check object exists in the pool after creation').to.exist; // inject OBJECT_DELETE for the object. this should tombstone the object and make it inaccessible to the end user, but still keep it in memory in the local pool - await objectsHelper.processStateOperationMessageOnChannel({ + await objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', @@ -4298,7 +4301,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const channelConfigurationScenarios = [ { - description: 'public API throws missing state modes error when attached without correct state modes', + description: 'public API throws missing object modes error when attached without correct modes', action: async (ctx) => { const { objects, channel, map, counter } = ctx; @@ -4320,7 +4323,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { description: - 'public API throws missing state modes error when not yet attached but client options are missing correct modes', + 'public API throws missing object modes error when not yet attached but client options are missing correct modes', action: async (ctx) => { const { objects, channel, map, counter, helper } = ctx; @@ -4454,7 +4457,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function * @spec TO3l8 * @spec RSL1i */ - it('state message publish respects connectionDetails.maxMessageSize', async function () { + it('object message publish respects connectionDetails.maxMessageSize', async function () { const helper = this.test.helper; const client = RealtimeWithObjects(helper, { clientId: 'test' }); @@ -4491,46 +4494,46 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const data = new Array(100).fill('a').join(''); const error = await expectToThrowAsync( async () => root.set('key', data), - 'Maximum size of state messages that can be published at once exceeded', + 'Maximum size of object messages that can be published at once exceeded', ); expect(error.code).to.equal(40009, 'Check maximum size of messages error has correct error code'); }, client); }); - describe('StateMessage message size', () => { - const stateMessageSizeScenarios = [ + describe('ObjectMessage message size', () => { + const objectMessageSizeScenarios = [ { description: 'client id', - message: stateMessageFromValues({ + message: objectMessageFromValues({ clientId: 'my-client', }), expected: Utils.dataSizeBytes('my-client'), }, { description: 'extras', - message: stateMessageFromValues({ + message: objectMessageFromValues({ extras: { foo: 'bar' }, }), expected: Utils.dataSizeBytes('{"foo":"bar"}'), }, { description: 'object id', - message: stateMessageFromValues({ + message: objectMessageFromValues({ operation: { objectId: 'object-id' }, }), expected: 0, }, { description: 'map create op no payload', - message: stateMessageFromValues({ + message: objectMessageFromValues({ operation: { action: 0, objectId: 'object-id' }, }), expected: 0, }, { description: 'map create op with object payload', - message: stateMessageFromValues( + message: objectMessageFromValues( { operation: { action: 0, @@ -4547,7 +4550,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { description: 'map create op with string payload', - message: stateMessageFromValues( + message: objectMessageFromValues( { operation: { action: 0, @@ -4561,7 +4564,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { description: 'map create op with bytes payload', - message: stateMessageFromValues( + message: objectMessageFromValues( { operation: { action: 0, @@ -4578,7 +4581,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { description: 'map create op with boolean payload', - message: stateMessageFromValues( + message: objectMessageFromValues( { operation: { action: 0, @@ -4592,14 +4595,14 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { description: 'map remove op', - message: stateMessageFromValues({ + message: objectMessageFromValues({ operation: { action: 2, objectId: 'object-id', mapOp: { key: 'my-key' } }, }), expected: Utils.dataSizeBytes('my-key'), }, { description: 'map set operation value=object', - message: stateMessageFromValues({ + message: objectMessageFromValues({ operation: { action: 1, objectId: 'object-id', @@ -4610,14 +4613,14 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { description: 'map set operation value=string', - message: stateMessageFromValues({ + message: objectMessageFromValues({ operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { value: 'my-value' } } }, }), expected: Utils.dataSizeBytes('my-key') + Utils.dataSizeBytes('my-value'), }, { description: 'map set operation value=bytes', - message: stateMessageFromValues({ + message: objectMessageFromValues({ operation: { action: 1, objectId: 'object-id', @@ -4628,21 +4631,21 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { description: 'map set operation value=boolean', - message: stateMessageFromValues({ + message: objectMessageFromValues({ operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { value: true } } }, }), expected: Utils.dataSizeBytes('my-key') + 1, }, { description: 'map set operation value=double', - message: stateMessageFromValues({ + message: objectMessageFromValues({ operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { value: 123.456 } } }, }), expected: Utils.dataSizeBytes('my-key') + 8, }, { description: 'map object', - message: stateMessageFromValues({ + message: objectMessageFromValues({ object: { objectId: 'object-id', map: { @@ -4671,28 +4674,28 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, { description: 'counter create op no payload', - message: stateMessageFromValues({ + message: objectMessageFromValues({ operation: { action: 3, objectId: 'object-id' }, }), expected: 0, }, { description: 'counter create op with payload', - message: stateMessageFromValues({ + message: objectMessageFromValues({ operation: { action: 3, objectId: 'object-id', counter: { count: 1234567 } }, }), expected: 8, }, { description: 'counter inc op', - message: stateMessageFromValues({ + message: objectMessageFromValues({ operation: { action: 4, objectId: 'object-id', counterOp: { amount: 123.456 } }, }), expected: 8, }, { description: 'counter object', - message: stateMessageFromValues({ + message: objectMessageFromValues({ object: { objectId: 'object-id', counter: { count: 1234567 }, @@ -4710,20 +4713,20 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ]; /** @nospec */ - forScenarios(this, stateMessageSizeScenarios, function (helper, scenario) { - helper.recordPrivateApi('call.StateMessage.encode'); - ObjectsPlugin.StateMessage.encode(scenario.message); + forScenarios(this, objectMessageSizeScenarios, function (helper, scenario) { + helper.recordPrivateApi('call.ObjectMessage.encode'); + ObjectsPlugin.ObjectMessage.encode(scenario.message); helper.recordPrivateApi('call.BufferUtils.utf8Encode'); // was called by a scenario to create buffers - helper.recordPrivateApi('call.StateMessage.fromValues'); // was called by a scenario to create a StateMessage instance + helper.recordPrivateApi('call.ObjectMessage.fromValues'); // was called by a scenario to create an ObjectMessage instance helper.recordPrivateApi('call.Utils.dataSizeBytes'); // was called by a scenario to calculated the expected byte size - helper.recordPrivateApi('call.StateMessage.getMessageSize'); + helper.recordPrivateApi('call.ObjectMessage.getMessageSize'); expect(scenario.message.getMessageSize()).to.equal(scenario.expected); }); }); }); /** @nospec */ - it('can attach to channel with Objects state modes', async function () { + it('can attach to channel with Objects modes', async function () { const helper = this.test.helper; const client = helper.AblyRealtime(); From 8846a8bed8d7922f3fc5a370ae36b40284346a90 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 27 Mar 2025 10:42:04 +0000 Subject: [PATCH 140/166] Change `statemessage.ts` filename to `objectmessage.ts` --- scripts/moduleReport.ts | 2 +- src/plugins/objects/batchcontext.ts | 2 +- src/plugins/objects/index.ts | 2 +- src/plugins/objects/livecounter.ts | 2 +- src/plugins/objects/livemap.ts | 8 ++++---- src/plugins/objects/liveobject.ts | 2 +- src/plugins/objects/{statemessage.ts => objectmessage.ts} | 0 src/plugins/objects/objects.ts | 2 +- src/plugins/objects/syncobjectsdatapool.ts | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) rename src/plugins/objects/{statemessage.ts => objectmessage.ts} (100%) diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index a79a6abd75..68d2a2373e 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -331,9 +331,9 @@ async function checkObjectsPluginFiles() { 'src/plugins/objects/livemap.ts', 'src/plugins/objects/liveobject.ts', 'src/plugins/objects/objectid.ts', + 'src/plugins/objects/objectmessage.ts', 'src/plugins/objects/objects.ts', 'src/plugins/objects/objectspool.ts', - 'src/plugins/objects/statemessage.ts', 'src/plugins/objects/syncobjectsdatapool.ts', ]); diff --git a/src/plugins/objects/batchcontext.ts b/src/plugins/objects/batchcontext.ts index 8194bba034..8e6bc2e76c 100644 --- a/src/plugins/objects/batchcontext.ts +++ b/src/plugins/objects/batchcontext.ts @@ -4,9 +4,9 @@ import { BatchContextLiveCounter } from './batchcontextlivecounter'; import { BatchContextLiveMap } from './batchcontextlivemap'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; +import { ObjectMessage } from './objectmessage'; import { Objects } from './objects'; import { ROOT_OBJECT_ID } from './objectspool'; -import { ObjectMessage } from './statemessage'; export class BatchContext { private _client: BaseClient; diff --git a/src/plugins/objects/index.ts b/src/plugins/objects/index.ts index e9144588c5..1b9d27f743 100644 --- a/src/plugins/objects/index.ts +++ b/src/plugins/objects/index.ts @@ -1,5 +1,5 @@ +import { ObjectMessage } from './objectmessage'; import { Objects } from './objects'; -import { ObjectMessage } from './statemessage'; export { Objects, ObjectMessage }; diff --git a/src/plugins/objects/livecounter.ts b/src/plugins/objects/livecounter.ts index 8cc8781f6f..d114ff21e4 100644 --- a/src/plugins/objects/livecounter.ts +++ b/src/plugins/objects/livecounter.ts @@ -1,7 +1,7 @@ import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { ObjectId } from './objectid'; +import { CounterOp, ObjectMessage, ObjectOperation, ObjectOperationAction, ObjectState } from './objectmessage'; import { Objects } from './objects'; -import { CounterOp, ObjectMessage, ObjectOperation, ObjectOperationAction, ObjectState } from './statemessage'; export interface LiveCounterData extends LiveObjectData { data: number; diff --git a/src/plugins/objects/livemap.ts b/src/plugins/objects/livemap.ts index 415e309b92..73655f586a 100644 --- a/src/plugins/objects/livemap.ts +++ b/src/plugins/objects/livemap.ts @@ -4,17 +4,17 @@ import type * as API from '../../../ably'; import { DEFAULTS } from './defaults'; import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; import { ObjectId } from './objectid'; -import { Objects } from './objects'; import { - MapSemantics, MapEntry, MapOp, + MapSemantics, ObjectMessage, - ObjectState, ObjectOperation, ObjectOperationAction, + ObjectState, ObjectValue, -} from './statemessage'; +} from './objectmessage'; +import { Objects } from './objects'; export interface ObjectIdObjectData { /** A reference to another object, used to support composable object structures. */ diff --git a/src/plugins/objects/liveobject.ts b/src/plugins/objects/liveobject.ts index ca5587e2c9..9ba9557244 100644 --- a/src/plugins/objects/liveobject.ts +++ b/src/plugins/objects/liveobject.ts @@ -1,7 +1,7 @@ import type BaseClient from 'common/lib/client/baseclient'; import type EventEmitter from 'common/lib/util/eventemitter'; +import { ObjectMessage, ObjectOperation, ObjectState } from './objectmessage'; import { Objects } from './objects'; -import { ObjectMessage, ObjectState, ObjectOperation } from './statemessage'; export enum LiveObjectSubscriptionEvent { updated = 'updated', diff --git a/src/plugins/objects/statemessage.ts b/src/plugins/objects/objectmessage.ts similarity index 100% rename from src/plugins/objects/statemessage.ts rename to src/plugins/objects/objectmessage.ts diff --git a/src/plugins/objects/objects.ts b/src/plugins/objects/objects.ts index a21401702b..a92319d265 100644 --- a/src/plugins/objects/objects.ts +++ b/src/plugins/objects/objects.ts @@ -7,8 +7,8 @@ import { DEFAULTS } from './defaults'; import { LiveCounter } from './livecounter'; import { LiveMap } from './livemap'; import { LiveObject, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; +import { ObjectMessage, ObjectOperationAction } from './objectmessage'; import { ObjectsPool, ROOT_OBJECT_ID } from './objectspool'; -import { ObjectMessage, ObjectOperationAction } from './statemessage'; import { SyncObjectsDataPool } from './syncobjectsdatapool'; export enum ObjectsEvent { diff --git a/src/plugins/objects/syncobjectsdatapool.ts b/src/plugins/objects/syncobjectsdatapool.ts index 478a7590fc..bb069adbe2 100644 --- a/src/plugins/objects/syncobjectsdatapool.ts +++ b/src/plugins/objects/syncobjectsdatapool.ts @@ -1,7 +1,7 @@ import type BaseClient from 'common/lib/client/baseclient'; import type RealtimeChannel from 'common/lib/client/realtimechannel'; +import { ObjectMessage, ObjectState } from './objectmessage'; import { Objects } from './objects'; -import { ObjectMessage, ObjectState } from './statemessage'; export interface LiveObjectDataEntry { objectState: ObjectState; From 84c1643972f5b67248bcf9492327e1e15f63af66 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 27 Mar 2025 12:43:30 +0000 Subject: [PATCH 141/166] Don't use server side terminology and knowledge regarding timeserials / site codes in client side LiveObjects implementation See the relevant discussion in the spec PR [1] [1] https://github.com/ably/specification/pull/279#discussion_r2003000877 --- src/plugins/objects/defaults.ts | 2 +- src/plugins/objects/livecounter.ts | 14 +++--- src/plugins/objects/livemap.ts | 74 ++++++++++++++-------------- src/plugins/objects/liveobject.ts | 24 ++++----- src/plugins/objects/objectmessage.ts | 10 ++-- src/plugins/objects/objects.ts | 18 +++---- 6 files changed, 71 insertions(+), 71 deletions(-) diff --git a/src/plugins/objects/defaults.ts b/src/plugins/objects/defaults.ts index f5fb9d5c4f..fa3a091014 100644 --- a/src/plugins/objects/defaults.ts +++ b/src/plugins/objects/defaults.ts @@ -2,7 +2,7 @@ export const DEFAULTS = { gcInterval: 1000 * 60 * 5, // 5 minutes /** * Must be > 2 minutes to ensure we keep tombstones long enough to avoid the possibility of receiving an operation - * with an earlier origin timeserial that would not have been applied if the tombstone still existed. + * with an earlier serial that would not have been applied if the tombstone still existed. * * Applies both for map entries tombstones and object tombstones. */ diff --git a/src/plugins/objects/livecounter.ts b/src/plugins/objects/livecounter.ts index d114ff21e4..ccc358bc2a 100644 --- a/src/plugins/objects/livecounter.ts +++ b/src/plugins/objects/livecounter.ts @@ -168,20 +168,20 @@ export class LiveCounter extends LiveObject ); } - const opOriginTimeserial = msg.serial!; + const opSerial = msg.serial!; const opSiteCode = msg.siteCode!; - if (!this._canApplyOperation(opOriginTimeserial, opSiteCode)) { + if (!this._canApplyOperation(opSerial, opSiteCode)) { this._client.Logger.logAction( this._client.logger, this._client.Logger.LOG_MICRO, 'LiveCounter.applyOperation()', - `skipping ${op.action} op: op timeserial ${opOriginTimeserial.toString()} <= site timeserial ${this._siteTimeserials[opSiteCode]?.toString()}; objectId=${this.getObjectId()}`, + `skipping ${op.action} op: op serial ${opSerial.toString()} <= site serial ${this._siteTimeserials[opSiteCode]?.toString()}; objectId=${this.getObjectId()}`, ); return; } - // should update stored site timeserial immediately. doesn't matter if we successfully apply the op, + // should update stored site serial immediately. doesn't matter if we successfully apply the op, // as it's important to mark that the op was processed by the object - this._siteTimeserials[opSiteCode] = opOriginTimeserial; + this._siteTimeserials[opSiteCode] = opSerial; if (this.isTombstoned()) { // this object is tombstoned so the operation cannot be applied @@ -250,8 +250,8 @@ export class LiveCounter extends LiveObject } } - // object's site timeserials are still updated even if it is tombstoned, so always use the site timeserials received from the op. - // should default to empty map if site timeserials do not exist on the object state, so that any future operation may be applied to this object. + // object's site serials are still updated even if it is tombstoned, so always use the site serials received from the operation. + // should default to empty map if site serials do not exist on the object state, so that any future operation may be applied to this object. this._siteTimeserials = objectState.siteTimeserials ?? {}; if (this.isTombstoned()) { diff --git a/src/plugins/objects/livemap.ts b/src/plugins/objects/livemap.ts index 73655f586a..0c6b44d185 100644 --- a/src/plugins/objects/livemap.ts +++ b/src/plugins/objects/livemap.ts @@ -36,7 +36,7 @@ export type ObjectData = ObjectIdObjectData | ValueObjectData; export interface LiveMapEntry { tombstone: boolean; /** - * Can't use timeserial from the operation that deleted the entry for the same reason as for {@link LiveObject} tombstones, see explanation there. + * Can't use serial from the operation that deleted the entry for the same reason as for {@link LiveObject} tombstones, see explanation there. */ tombstonedAt: number | undefined; timeserial: string | undefined; @@ -371,20 +371,20 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject { - // for MAP_CREATE op we must use dedicated timeserial field available on an entry, instead of a timeserial on a message - const opOriginTimeserial = entry.timeserial; + // for a MAP_CREATE operation we must use the serial value available on an entry, instead of a serial on a message + const opSerial = entry.timeserial; let update: LiveMapUpdate | LiveObjectUpdateNoop; if (entry.tombstone === true) { // entry in MAP_CREATE op is removed, try to apply MAP_REMOVE op - update = this._applyMapRemove({ key }, opOriginTimeserial); + update = this._applyMapRemove({ key }, opSerial); } else { // entry in MAP_CREATE op is not removed, try to set it via MAP_SET op - update = this._applyMapSet({ key, data: entry.data }, opOriginTimeserial); + update = this._applyMapSet({ key, data: entry.data }, opSerial); } // skip noop updates @@ -649,17 +649,17 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject extends LiveObject extends LiveObject entryTimeserial; + // if both serials exist, compare them lexicographically + return opSerial > mapEntrySerial; } private _liveMapDataFromMapEntries(entries: Record): LiveMapData { diff --git a/src/plugins/objects/liveobject.ts b/src/plugins/objects/liveobject.ts index 9ba9557244..3a12114547 100644 --- a/src/plugins/objects/liveobject.ts +++ b/src/plugins/objects/liveobject.ts @@ -52,8 +52,8 @@ export abstract class LiveObject< protected _createOperationIsMerged: boolean; private _tombstone: boolean; /** - * Even though the `timeserial` from the operation that deleted the object contains the timestamp value, - * the `timeserial` should be treated as an opaque string on the client, meaning we should not attempt to parse it. + * Even though the {@link ObjectMessage.serial} value from the operation that deleted the object contains the timestamp value, + * the serial should be treated as an opaque string on the client, meaning we should not attempt to parse it. * * Therefore, we need to set our own timestamp using local clock when the object is deleted client-side. * Strictly speaking, this does make an assumption about the client clock not being too heavily skewed behind the server, @@ -70,7 +70,7 @@ export abstract class LiveObject< this._lifecycleEvents = new this._client.EventEmitter(this._client.logger); this._objectId = objectId; this._dataRef = this._getZeroValueData(); - // use empty timeserials vector by default, so any future operation can be applied to this object + // use empty map of serials by default, so any future operation can be applied to this object this._siteTimeserials = {}; this._createOperationIsMerged = false; this._tombstone = false; @@ -181,22 +181,22 @@ export abstract class LiveObject< } /** - * Returns true if the given origin timeserial indicates that the operation to which it belongs should be applied to the object. + * Returns true if the given serial indicates that the operation to which it belongs should be applied to the object. * - * An operation should be applied if the origin timeserial is strictly greater than the timeserial in the site timeserials for the same site. - * If the site timeserials do not contain a timeserial for the site of the origin timeserial, the operation should be applied. + * An operation should be applied if its serial is strictly greater than the serial in the `siteTimeserials` map for the same site. + * If `siteTimeserials` map does not contain a serial for the same site, the operation should be applied. */ - protected _canApplyOperation(opOriginTimeserial: string | undefined, opSiteCode: string | undefined): boolean { - if (!opOriginTimeserial) { - throw new this._client.ErrorInfo(`Invalid timeserial: ${opOriginTimeserial}`, 92000, 500); + protected _canApplyOperation(opSerial: string | undefined, opSiteCode: string | undefined): boolean { + if (!opSerial) { + throw new this._client.ErrorInfo(`Invalid serial: ${opSerial}`, 92000, 500); } if (!opSiteCode) { throw new this._client.ErrorInfo(`Invalid site code: ${opSiteCode}`, 92000, 500); } - const siteTimeserial = this._siteTimeserials[opSiteCode]; - return !siteTimeserial || opOriginTimeserial > siteTimeserial; + const siteSerial = this._siteTimeserials[opSiteCode]; + return !siteSerial || opSerial > siteSerial; } protected _applyObjectDelete(): TUpdate { @@ -216,7 +216,7 @@ export abstract class LiveObject< * Provided object state should hold a valid data for current LiveObject, e.g. counter data for LiveCounter, map data for LiveMap. * * Object states are received during sync sequence, and sync sequence is a source of truth for the current state of the objects, - * so we can use the data received from the sync sequence directly and override any data values or site timeserials this LiveObject has + * so we can use the data received from the sync sequence directly and override any data values or site serials this LiveObject has * without the need to merge them. * * Returns an update object that describes the changes applied based on the object's previous value. diff --git a/src/plugins/objects/objectmessage.ts b/src/plugins/objects/objectmessage.ts index f97cc912a1..983df96488 100644 --- a/src/plugins/objects/objectmessage.ts +++ b/src/plugins/objects/objectmessage.ts @@ -54,10 +54,10 @@ export interface MapEntry { /** Indicates whether the map entry has been removed. */ tombstone?: boolean; /** - * The *origin* timeserial of the last operation that was applied to the map entry. + * The {@link ObjectMessage.serial} value of the last operation that was applied to the map entry. * * It is optional in a MAP_CREATE operation and might be missing, in which case the client should use a nullish value for it - * and treat it as the "earliest possible" timeserial for comparison purposes. + * and treat it as the "earliest possible" serial for comparison purposes. */ timeserial?: string; /** The data that represents the value of the map entry. */ @@ -118,7 +118,7 @@ export interface ObjectOperation { export interface ObjectState { /** The identifier of the object. */ objectId: string; - /** A vector of origin timeserials keyed by site code of the last operation that was applied to this object. */ + /** A map of serials keyed by a {@link ObjectMessage.siteCode}, representing the last operations applied to this object */ siteTimeserials: Record; /** True if the object has been tombstoned. */ tombstone: boolean; @@ -162,9 +162,9 @@ export class ObjectMessage { * Mutually exclusive with the `operation` field. This field is only set on object messages if the `action` field of the `ProtocolMessage` encapsulating it is `OBJECT_SYNC`. */ object?: ObjectState; - /** Timeserial format. Contains the origin timeserial for this object message. */ + /** An opaque string that uniquely identifies this object message. */ serial?: string; - /** Site code corresponding to this message's timeserial */ + /** An opaque string used as a key to update the map of serial values on an object. */ siteCode?: string; constructor( diff --git a/src/plugins/objects/objects.ts b/src/plugins/objects/objects.ts index a92319d265..9914c607ea 100644 --- a/src/plugins/objects/objects.ts +++ b/src/plugins/objects/objects.ts @@ -114,16 +114,16 @@ export class Objects { await this.publish([msg]); - // we may have already received the CREATE operation at this point, as it could arrive before the ACK for our publish message. - // this means the object might already exist in the local pool, having been added during the usual CREATE operation process. + // we may have already received the MAP_CREATE operation at this point, as it could arrive before the ACK for our publish message. + // this means the object might already exist in the local pool, having been added during the usual MAP_CREATE operation process. // here we check if the object is present, and return it if found; otherwise, create a new object on the client side. if (this._objectsPool.get(objectId)) { return this._objectsPool.get(objectId) as LiveMap; } - // we haven't received the CREATE operation yet, so we can create a new map object using the locally constructed object operation. - // we don't know the timeserials for map entries, so we assign an "earliest possible" timeserial to each entry, so that any subsequent operation can be applied to them. - // we mark the CREATE operation as merged for the object, guaranteeing its idempotency and preventing it from being applied again when the operation arrives. + // we haven't received the MAP_CREATE operation yet, so we can create a new map object using the locally constructed object operation. + // we don't know the serials for map entries, so we assign an "earliest possible" serial to each entry, so that any subsequent operation can be applied to them. + // we mark the MAP_CREATE operation as merged for the object, guaranteeing its idempotency and preventing it from being applied again when the operation arrives. const map = LiveMap.fromObjectOperation(this, msg.operation!); this._objectsPool.set(objectId, map); @@ -146,15 +146,15 @@ export class Objects { await this.publish([msg]); - // we may have already received the CREATE operation at this point, as it could arrive before the ACK for our publish message. - // this means the object might already exist in the local pool, having been added during the usual CREATE operation process. + // we may have already received the COUNTER_CREATE operation at this point, as it could arrive before the ACK for our publish message. + // this means the object might already exist in the local pool, having been added during the usual COUNTER_CREATE operation process. // here we check if the object is present, and return it if found; otherwise, create a new object on the client side. if (this._objectsPool.get(objectId)) { return this._objectsPool.get(objectId) as LiveCounter; } - // we haven't received the CREATE operation yet, so we can create a new counter object using the locally constructed object operation. - // we mark the CREATE operation as merged for the object, guaranteeing its idempotency. this ensures we don't double count the initial counter value when the operation arrives. + // we haven't received the COUNTER_CREATE operation yet, so we can create a new counter object using the locally constructed object operation. + // we mark the COUNTER_CREATE operation as merged for the object, guaranteeing its idempotency. this ensures we don't double count the initial counter value when the operation arrives. const counter = LiveCounter.fromObjectOperation(this, msg.operation!); this._objectsPool.set(objectId, counter); From fef0155b37e398bb91b752cd735187ea9a34578f Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 9 Apr 2025 08:47:20 +0100 Subject: [PATCH 142/166] Add `@experimental` tags to LiveObjects methods Resolves PUB-1580 --- ably.d.ts | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/ably.d.ts b/ably.d.ts index 3b48903d1c..51b6274c39 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -2113,6 +2113,7 @@ export declare interface Objects { * ``` * * @returns A promise which, upon success, will be fulfilled with a {@link LiveMap} object. Upon failure, the promise will be rejected with an {@link ErrorInfo} object which explains the error. + * @experimental */ getRoot(): Promise>; @@ -2121,6 +2122,7 @@ export declare interface Objects { * * @param entries - The initial entries for the new {@link LiveMap} object. * @returns A promise which, upon success, will be fulfilled with a {@link LiveMap} object. Upon failure, the promise will be rejected with an {@link ErrorInfo} object which explains the error. + * @experimental */ createMap(entries?: T): Promise>; @@ -2129,6 +2131,7 @@ export declare interface Objects { * * @param count - The initial value for the new {@link LiveCounter} object. * @returns A promise which, upon success, will be fulfilled with a {@link LiveCounter} object. Upon failure, the promise will be rejected with an {@link ErrorInfo} object which explains the error. + * @experimental */ createCounter(count?: number): Promise; @@ -2144,6 +2147,7 @@ export declare interface Objects { * * @param callback - A batch callback function used to group operations together. Cannot be an `async` function. * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + * @experimental */ batch(callback: BatchCallback): Promise; @@ -2153,6 +2157,7 @@ export declare interface Objects { * @param event - The named event to listen for. * @param callback - The event listener. * @returns A {@link OnObjectsEventResponse} object that allows the provided listener to be deregistered from future updates. + * @experimental */ on(event: ObjectsEvent, callback: ObjectsEventCallback): OnObjectsEventResponse; @@ -2161,11 +2166,14 @@ export declare interface Objects { * * @param event - The named event. * @param callback - The event listener. + * @experimental */ off(event: ObjectsEvent, callback: ObjectsEventCallback): void; /** * Deregisters all registrations, for all events and listeners. + * + * @experimental */ offAll(): void; } @@ -2207,6 +2215,8 @@ export type DefaultRoot = export declare interface OnObjectsEventResponse { /** * Deregisters the listener passed to the `on` call. + * + * @experimental */ off(): void; } @@ -2219,6 +2229,7 @@ export declare interface BatchContext { * Mirrors the {@link Objects.getRoot} method and returns a {@link BatchContextLiveMap} wrapper for the root object on a channel. * * @returns A {@link BatchContextLiveMap} object. + * @experimental */ getRoot(): BatchContextLiveMap; } @@ -2232,11 +2243,14 @@ export declare interface BatchContextLiveMap { * * @param key - The key to retrieve the value for. * @returns A {@link LiveObject}, a primitive type (string, number, boolean, or binary data) or `undefined` if the key doesn't exist in a map or the associated {@link LiveObject} has been deleted. Always `undefined` if this map object is deleted. + * @experimental */ get(key: TKey): T[TKey] | undefined; /** * Returns the number of key-value pairs in the map. + * + * @experimental */ size(): number; @@ -2249,6 +2263,7 @@ export declare interface BatchContextLiveMap { * * @param key - The key to set the value for. * @param value - The value to assign to the key. + * @experimental */ set(key: TKey, value: T[TKey]): void; @@ -2260,6 +2275,7 @@ export declare interface BatchContextLiveMap { * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. * * @param key - The key to set the value for. + * @experimental */ remove(key: TKey): void; } @@ -2270,6 +2286,8 @@ export declare interface BatchContextLiveMap { export declare interface BatchContextLiveCounter { /** * Returns the current value of the counter. + * + * @experimental */ value(): number; @@ -2281,6 +2299,7 @@ export declare interface BatchContextLiveCounter { * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. * * @param amount - The amount by which to increase the counter value. + * @experimental */ increment(amount: number): void; @@ -2288,6 +2307,7 @@ export declare interface BatchContextLiveCounter { * An alias for calling {@link BatchContextLiveCounter.increment | BatchContextLiveCounter.increment(-amount)} * * @param amount - The amount by which to decrease the counter value. + * @experimental */ decrement(amount: number): void; } @@ -2307,26 +2327,35 @@ export declare interface LiveMap extends LiveObject(key: TKey): T[TKey] | undefined; /** * Returns the number of key-value pairs in the map. + * + * @experimental */ size(): number; /** * Returns an iterable of key-value pairs for every entry in the map. + * + * @experimental */ entries(): IterableIterator<[TKey, T[TKey]]>; /** * Returns an iterable of keys in the map. + * + * @experimental */ keys(): IterableIterator; /** * Returns an iterable of values in the map. + * + * @experimental */ values(): IterableIterator; @@ -2340,6 +2369,7 @@ export declare interface LiveMap extends LiveObject(key: TKey, value: T[TKey]): Promise; @@ -2352,6 +2382,7 @@ export declare interface LiveMap extends LiveObject(key: TKey): Promise; } @@ -2381,6 +2412,8 @@ export type ObjectValue = string | number | boolean | Buffer | ArrayBuffer; export declare interface LiveCounter extends LiveObject { /** * Returns the current value of the counter. + * + * @experimental */ value(): number; @@ -2393,6 +2426,7 @@ export declare interface LiveCounter extends LiveObject { * * @param amount - The amount by which to increase the counter value. * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + * @experimental */ increment(amount: number): Promise; @@ -2401,6 +2435,7 @@ export declare interface LiveCounter extends LiveObject { * * @param amount - The amount by which to decrease the counter value. * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + * @experimental */ decrement(amount: number): Promise; } @@ -2429,6 +2464,7 @@ export declare interface LiveObject): SubscribeResponse; @@ -2436,11 +2472,14 @@ export declare interface LiveObject): void; /** * Deregisters all listeners from updates for this LiveObject. + * + * @experimental */ unsubscribeAll(): void; @@ -2450,6 +2489,7 @@ export declare interface LiveObject Date: Wed, 9 Apr 2025 08:27:08 +0100 Subject: [PATCH 143/166] Infer key name of the LiveMap in its direct subscription update object Resolves PUB-1094 --- ably.d.ts | 6 +- src/plugins/objects/livemap.ts | 57 +++++++++++-------- .../browser/template/src/index-objects.ts | 7 ++- 3 files changed, 42 insertions(+), 28 deletions(-) diff --git a/ably.d.ts b/ably.d.ts index 3b48903d1c..4a416ea1b0 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -2299,7 +2299,7 @@ export declare interface BatchContextLiveCounter { * * Keys must be strings. Values can be another {@link LiveObject}, or a primitive type, such as a string, number, boolean, or binary data (see {@link ObjectValue}). */ -export declare interface LiveMap extends LiveObject { +export declare interface LiveMap extends LiveObject> { /** * Returns the value associated with a given key. Returns `undefined` if the key doesn't exist in a map or if the associated {@link LiveObject} has been deleted. * @@ -2359,13 +2359,13 @@ export declare interface LiveMap extends LiveObject extends LiveObjectUpdate { /** * An object containing keys from a `LiveMap` that have changed, along with their change status: * - `updated` - the value of a key in the map was updated. * - `removed` - the key was removed from the map. */ - update: { [keyName: string]: 'updated' | 'removed' }; + update: { [keyName in keyof T & string]?: 'updated' | 'removed' }; } /** diff --git a/src/plugins/objects/livemap.ts b/src/plugins/objects/livemap.ts index 0c6b44d185..90f4d7d023 100644 --- a/src/plugins/objects/livemap.ts +++ b/src/plugins/objects/livemap.ts @@ -47,11 +47,11 @@ export interface LiveMapData extends LiveObjectData { data: Map; } -export interface LiveMapUpdate extends LiveObjectUpdate { - update: { [keyName: string]: 'updated' | 'removed' }; +export interface LiveMapUpdate extends LiveObjectUpdate { + update: { [keyName in keyof T & string]?: 'updated' | 'removed' }; } -export class LiveMap extends LiveObject { +export class LiveMap extends LiveObject> { constructor( objects: Objects, private _semantics: MapSemantics, @@ -391,7 +391,7 @@ export class LiveMap extends LiveObject | LiveObjectUpdateNoop; switch (op.action) { case ObjectOperationAction.MAP_CREATE: update = this._applyMapCreate(op); @@ -435,7 +435,7 @@ export class LiveMap extends LiveObject | LiveObjectUpdateNoop { if (objectState.objectId !== this.getObjectId()) { throw new this._client.ErrorInfo( `Invalid object state: object state objectId=${objectState.objectId}; LiveMap objectId=${this.getObjectId()}`, @@ -526,21 +526,23 @@ export class LiveMap extends LiveObject() }; } - protected _updateFromDataDiff(prevDataRef: LiveMapData, newDataRef: LiveMapData): LiveMapUpdate { - const update: LiveMapUpdate = { update: {} }; + protected _updateFromDataDiff(prevDataRef: LiveMapData, newDataRef: LiveMapData): LiveMapUpdate { + const update: LiveMapUpdate = { update: {} }; for (const [key, currentEntry] of prevDataRef.data.entries()) { + const typedKey: keyof T & string = key; // any non-tombstoned properties that exist on a current map, but not in the new data - got removed - if (currentEntry.tombstone === false && !newDataRef.data.has(key)) { - update.update[key] = 'removed'; + if (currentEntry.tombstone === false && !newDataRef.data.has(typedKey)) { + update.update[typedKey] = 'removed'; } } for (const [key, newEntry] of newDataRef.data.entries()) { - if (!prevDataRef.data.has(key)) { + const typedKey: keyof T & string = key; + if (!prevDataRef.data.has(typedKey)) { // if property does not exist in the current map, but new data has it as a non-tombstoned property - got updated if (newEntry.tombstone === false) { - update.update[key] = 'updated'; + update.update[typedKey] = 'updated'; continue; } @@ -551,17 +553,17 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject { if (this._client.Utils.isNil(objectOperation.map)) { // if a map object is missing for the MAP_CREATE op, the initial value is implicitly an empty map. // in this case there is nothing to merge into the current map, so we can just end processing the op. return { update: {} }; } - const aggregatedUpdate: LiveMapUpdate | LiveObjectUpdateNoop = { update: {} }; + const aggregatedUpdate: LiveMapUpdate = { update: {} }; // in order to apply MAP_CREATE op for an existing map, we should merge their underlying entries keys. // we can do this by iterating over entries from MAP_CREATE op and apply changes on per-key basis as if we had MAP_SET, MAP_REMOVE operations. Object.entries(objectOperation.map.entries ?? {}).forEach(([key, entry]) => { // for a MAP_CREATE operation we must use the serial value available on an entry, instead of a serial on a message const opSerial = entry.timeserial; - let update: LiveMapUpdate | LiveObjectUpdateNoop; + let update: LiveMapUpdate | LiveObjectUpdateNoop; if (entry.tombstone === true) { // entry in MAP_CREATE op is removed, try to apply MAP_REMOVE op update = this._applyMapRemove({ key }, opSerial); @@ -624,7 +626,7 @@ export class LiveMap extends LiveObject | LiveObjectUpdateNoop { if (this._createOperationIsMerged) { // There can't be two different create operation for the same object id, because the object id // fully encodes that operation. This means we can safely ignore any new incoming create operations @@ -649,7 +651,7 @@ export class LiveMap extends LiveObject | LiveObjectUpdateNoop { const { ErrorInfo, Utils } = this._client; const existingEntry = this._dataRef.data.get(op.key); @@ -698,10 +700,15 @@ export class LiveMap extends LiveObject = { update: {} }; + const typedKey: keyof T & string = op.key; + update.update[typedKey] = 'updated'; + + return update; } - private _applyMapRemove(op: MapOp, opSerial: string | undefined): LiveMapUpdate | LiveObjectUpdateNoop { + private _applyMapRemove(op: MapOp, opSerial: string | undefined): LiveMapUpdate | LiveObjectUpdateNoop { const existingEntry = this._dataRef.data.get(op.key); if (existingEntry && !this._canApplyMapOperation(existingEntry.timeserial, opSerial)) { // the operation's serial <= the entry's serial, ignore the operation. @@ -729,7 +736,11 @@ export class LiveMap extends LiveObject = { update: {} }; + const typedKey: keyof T & string = op.key; + update.update[typedKey] = 'removed'; + + return update; } /** diff --git a/test/package/browser/template/src/index-objects.ts b/test/package/browser/template/src/index-objects.ts index bed8e14216..3796c529e2 100644 --- a/test/package/browser/template/src/index-objects.ts +++ b/test/package/browser/template/src/index-objects.ts @@ -42,13 +42,16 @@ globalThis.testAblyPackage = async function () { // check LiveMap subscription callback has correct TypeScript types const { unsubscribe } = root.subscribe(({ update }) => { - switch (update.someKey) { + // check update object infers keys from map type + const typedKeyOnMap = update.stringKey; + switch (typedKeyOnMap) { case 'removed': case 'updated': + case undefined: break; default: // check all possible types are exhausted - const shouldExhaustAllTypes: never = update.someKey; + const shouldExhaustAllTypes: never = typedKeyOnMap; } }); unsubscribe(); From 2689b479668609ec3207f33a201d0a252c18e900 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 26 Mar 2025 09:01:15 +0000 Subject: [PATCH 144/166] Update ably-js objects tests to use new REST API for Objects Update to use new API from [1] Resolves PUB-1530 [1] https://ably.atlassian.net/browse/PUB-1191 --- test/common/modules/objects_helper.js | 160 +++++++++++++---- test/realtime/objects.test.js | 248 ++++++++++++++------------ 2 files changed, 255 insertions(+), 153 deletions(-) diff --git a/test/common/modules/objects_helper.js b/test/common/modules/objects_helper.js index 1d3dacf973..ff8e48b96e 100644 --- a/test/common/modules/objects_helper.js +++ b/test/common/modules/objects_helper.js @@ -14,6 +14,14 @@ define(['ably', 'shared_helper', 'objects'], function (Ably, Helper, ObjectsPlug COUNTER_INC: 4, OBJECT_DELETE: 5, }; + const ACTION_STRINGS = { + MAP_CREATE: 'MAP_CREATE', + MAP_SET: 'MAP_SET', + MAP_REMOVE: 'MAP_REMOVE', + COUNTER_CREATE: 'COUNTER_CREATE', + COUNTER_INC: 'COUNTER_INC', + OBJECT_DELETE: 'OBJECT_DELETE', + }; function nonce() { return Helper.randomString(); @@ -45,61 +53,49 @@ define(['ably', 'shared_helper', 'objects'], function (Ably, Helper, ObjectsPlug const emptyCounter = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'emptyCounter', - createOp: this.counterCreateOp(), + createOp: this.counterCreateRestOp(), }); const initialValueCounter = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'initialValueCounter', - createOp: this.counterCreateOp({ count: 10 }), + createOp: this.counterCreateRestOp({ number: 10 }), }); const referencedCounter = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'referencedCounter', - createOp: this.counterCreateOp({ count: 20 }), + createOp: this.counterCreateRestOp({ number: 20 }), }); const emptyMap = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'emptyMap', - createOp: this.mapCreateOp(), + createOp: this.mapCreateRestOp(), }); const referencedMap = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'referencedMap', - createOp: this.mapCreateOp({ entries: { counterKey: { data: { objectId: referencedCounter.objectId } } } }), + createOp: this.mapCreateRestOp({ data: { counterKey: { objectId: referencedCounter.objectId } } }), }); const valuesMap = await this.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'valuesMap', - createOp: this.mapCreateOp({ - entries: { - stringKey: { data: { value: 'stringValue' } }, - emptyStringKey: { data: { value: '' } }, - bytesKey: { - data: { value: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9', encoding: 'base64' }, - }, - emptyBytesKey: { data: { value: '', encoding: 'base64' } }, - numberKey: { data: { value: 1 } }, - zeroKey: { data: { value: 0 } }, - trueKey: { data: { value: true } }, - falseKey: { data: { value: false } }, - mapKey: { data: { objectId: referencedMap.objectId } }, + createOp: this.mapCreateRestOp({ + data: { + stringKey: { string: 'stringValue' }, + emptyStringKey: { string: '' }, + bytesKey: { bytes: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9' }, + emptyBytesKey: { bytes: '' }, + numberKey: { number: 1 }, + zeroKey: { number: 0 }, + trueKey: { boolean: true }, + falseKey: { boolean: false }, + mapKey: { objectId: referencedMap.objectId }, }, }), }); } - async createAndSetOnMap(channelName, opts) { - const { mapObjectId, key, createOp } = opts; - - const createResult = await this.stateRequest(channelName, createOp); - await this.stateRequest( - channelName, - this.mapSetOp({ objectId: mapObjectId, key, data: { objectId: createResult.objectId } }), - ); - - return createResult; - } + // #region Wire Object Messages mapCreateOp(opts) { const { objectId, entries } = opts ?? {}; @@ -297,17 +293,97 @@ define(['ably', 'shared_helper', 'objects'], function (Ably, Helper, ObjectsPlug ); } - fakeMapObjectId() { - return `map:${Helper.randomString()}@${Date.now()}`; + // #endregion + + // #region REST API Operations + + async createAndSetOnMap(channelName, opts) { + const { mapObjectId, key, createOp } = opts; + + const createResult = await this.operationRequest(channelName, createOp); + const objectId = createResult.objectId; + await this.operationRequest(channelName, this.mapSetRestOp({ objectId: mapObjectId, key, value: { objectId } })); + + return createResult; } - fakeCounterObjectId() { - return `counter:${Helper.randomString()}@${Date.now()}`; + mapCreateRestOp(opts) { + const { objectId, nonce, data } = opts ?? {}; + const opBody = { + operation: ACTION_STRINGS.MAP_CREATE, + }; + + if (data) { + opBody.data = data; + } + + if (objectId != null) { + opBody.objectId = objectId; + opBody.nonce = nonce; + } + + return opBody; } - async stateRequest(channelName, opBody) { + mapSetRestOp(opts) { + const { objectId, key, value } = opts ?? {}; + const opBody = { + operation: ACTION_STRINGS.MAP_SET, + objectId, + data: { + key, + value, + }, + }; + + return opBody; + } + + mapRemoveRestOp(opts) { + const { objectId, key } = opts ?? {}; + const opBody = { + operation: ACTION_STRINGS.MAP_REMOVE, + objectId, + data: { + key, + }, + }; + + return opBody; + } + + counterCreateRestOp(opts) { + const { objectId, nonce, number } = opts ?? {}; + const opBody = { + operation: ACTION_STRINGS.COUNTER_CREATE, + }; + + if (number != null) { + opBody.data = { number }; + } + + if (objectId != null) { + opBody.objectId = objectId; + opBody.nonce = nonce; + } + + return opBody; + } + + counterIncRestOp(opts) { + const { objectId, number } = opts ?? {}; + const opBody = { + operation: ACTION_STRINGS.COUNTER_INC, + objectId, + data: { number }, + }; + + return opBody; + } + + async operationRequest(channelName, opBody) { if (Array.isArray(opBody)) { - throw new Error(`Only single object state requests are supported`); + throw new Error(`Only single object operation requests are supported`); } const method = 'post'; @@ -316,9 +392,9 @@ define(['ably', 'shared_helper', 'objects'], function (Ably, Helper, ObjectsPlug const response = await this._rest.request(method, path, 3, null, opBody, null); if (response.success) { - // only one operation in request, so need only first item. + // only one operation in the request, so need only the first item. const result = response.items[0]; - // extract object id if present + // extract objectId if present result.objectId = result.objectIds?.[0]; return result; } @@ -327,6 +403,16 @@ define(['ably', 'shared_helper', 'objects'], function (Ably, Helper, ObjectsPlug `${method}: ${path} FAILED; http code = ${response.statusCode}, error code = ${response.errorCode}, message = ${response.errorMessage}; operation = ${JSON.stringify(opBody)}`, ); } + + // #endregion + + fakeMapObjectId() { + return `map:${Helper.randomString()}@${Date.now()}`; + } + + fakeCounterObjectId() { + return `counter:${Helper.randomString()}@${Date.now()}`; + } } return (module.exports = ObjectsHelper); diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js index fc892e44b6..e2e52caaf5 100644 --- a/test/realtime/objects.test.js +++ b/test/realtime/objects.test.js @@ -623,19 +623,28 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ); const primitiveKeyData = [ - { key: 'stringKey', data: { value: 'stringValue' } }, - { key: 'emptyStringKey', data: { value: '' } }, + { key: 'stringKey', data: { value: 'stringValue' }, restData: { string: 'stringValue' } }, + { key: 'emptyStringKey', data: { value: '' }, restData: { string: '' } }, { key: 'bytesKey', data: { value: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9', encoding: 'base64' }, + restData: { bytes: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9' }, }, - { key: 'emptyBytesKey', data: { value: '', encoding: 'base64' } }, - { key: 'maxSafeIntegerKey', data: { value: Number.MAX_SAFE_INTEGER } }, - { key: 'negativeMaxSafeIntegerKey', data: { value: -Number.MAX_SAFE_INTEGER } }, - { key: 'numberKey', data: { value: 1 } }, - { key: 'zeroKey', data: { value: 0 } }, - { key: 'trueKey', data: { value: true } }, - { key: 'falseKey', data: { value: false } }, + { key: 'emptyBytesKey', data: { value: '', encoding: 'base64' }, restData: { bytes: '' } }, + { + key: 'maxSafeIntegerKey', + data: { value: Number.MAX_SAFE_INTEGER }, + restData: { number: Number.MAX_SAFE_INTEGER }, + }, + { + key: 'negativeMaxSafeIntegerKey', + data: { value: -Number.MAX_SAFE_INTEGER }, + restData: { number: -Number.MAX_SAFE_INTEGER }, + }, + { key: 'numberKey', data: { value: 1 }, restData: { number: 1 } }, + { key: 'zeroKey', data: { value: 0 }, restData: { number: 0 } }, + { key: 'trueKey', data: { value: true }, restData: { boolean: true } }, + { key: 'falseKey', data: { value: false }, restData: { boolean: false } }, ]; const primitiveMapsFixtures = [ { name: 'emptyMap' }, @@ -645,6 +654,10 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function acc[v.key] = { data: v.data }; return acc; }, {}), + restData: primitiveKeyData.reduce((acc, v) => { + acc[v.key] = v.restData; + return acc; + }, {}), }, ]; const countersFixtures = [ @@ -720,7 +733,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const { objectId: counterId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: objectsHelper.counterCreateOp({ count: 1 }), + createOp: objectsHelper.counterCreateRestOp({ number: 1 }), }); await counterCreatedPromise; @@ -773,7 +786,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const { objectId: counterId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: objectsHelper.counterCreateOp({ count: 1 }), + createOp: objectsHelper.counterCreateRestOp({ number: 1 }), }); await counterCreatedPromise; @@ -844,7 +857,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: fixture.name, - createOp: objectsHelper.mapCreateOp({ entries: fixture.entries }), + createOp: objectsHelper.mapCreateRestOp({ data: fixture.restData }), }), ), ); @@ -903,21 +916,21 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const mapCreatedPromise = waitForMapKeyUpdate(root, withReferencesMapKey); // create map with references. need to create referenced objects first to obtain their object ids - const { objectId: referencedMapObjectId } = await objectsHelper.stateRequest( + const { objectId: referencedMapObjectId } = await objectsHelper.operationRequest( channelName, - objectsHelper.mapCreateOp({ entries: { stringKey: { data: { value: 'stringValue' } } } }), + objectsHelper.mapCreateRestOp({ data: { stringKey: { string: 'stringValue' } } }), ); - const { objectId: referencedCounterObjectId } = await objectsHelper.stateRequest( + const { objectId: referencedCounterObjectId } = await objectsHelper.operationRequest( channelName, - objectsHelper.counterCreateOp({ count: 1 }), + objectsHelper.counterCreateRestOp({ number: 1 }), ); await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: withReferencesMapKey, - createOp: objectsHelper.mapCreateOp({ - entries: { - mapReference: { data: { objectId: referencedMapObjectId } }, - counterReference: { data: { objectId: referencedCounterObjectId } }, + createOp: objectsHelper.mapCreateRestOp({ + data: { + mapReference: { objectId: referencedMapObjectId }, + counterReference: { objectId: referencedCounterObjectId }, }, }), }); @@ -1060,12 +1073,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // apply MAP_SET ops await Promise.all( primitiveKeyData.map((keyData) => - objectsHelper.stateRequest( + objectsHelper.operationRequest( channelName, - objectsHelper.mapSetOp({ + objectsHelper.mapSetRestOp({ objectId: 'root', key: keyData.key, - data: keyData.data, + value: keyData.restData, }), ), ), @@ -1113,15 +1126,15 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'keyToCounter', - createOp: objectsHelper.counterCreateOp({ count: 1 }), + createOp: objectsHelper.counterCreateRestOp({ number: 1 }), }); await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'keyToMap', - createOp: objectsHelper.mapCreateOp({ - entries: { - stringKey: { data: { value: 'stringValue' } }, + createOp: objectsHelper.mapCreateRestOp({ + data: { + stringKey: { string: 'stringValue' }, }, }), }); @@ -1230,10 +1243,10 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const { objectId: mapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: mapKey, - createOp: objectsHelper.mapCreateOp({ - entries: { - shouldStay: { data: { value: 'foo' } }, - shouldDelete: { data: { value: 'bar' } }, + createOp: objectsHelper.mapCreateRestOp({ + data: { + shouldStay: { string: 'foo' }, + shouldDelete: { string: 'bar' }, }, }), }); @@ -1256,9 +1269,9 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const keyRemovedPromise = waitForMapKeyUpdate(map, 'shouldDelete'); // send MAP_REMOVE op - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapRemoveOp({ + objectsHelper.mapRemoveRestOp({ objectId: mapObjectId, key: 'shouldDelete', }), @@ -1377,7 +1390,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: fixture.name, - createOp: objectsHelper.counterCreateOp({ count: fixture.count }), + createOp: objectsHelper.counterCreateRestOp({ number: fixture.count }), }), ), ); @@ -1487,7 +1500,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: counterKey, - createOp: objectsHelper.counterCreateOp({ count: expectedCounterValue }), + createOp: objectsHelper.counterCreateRestOp({ number: expectedCounterValue }), }); await counterCreated; @@ -1521,11 +1534,11 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function expectedCounterValue += increment; const counterUpdatedPromise = waitForCounterUpdate(counter); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.counterIncOp({ + objectsHelper.counterIncRestOp({ objectId: counterObjectId, - amount: increment, + number: increment, }), ); await counterUpdatedPromise; @@ -1597,12 +1610,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const { objectId: mapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: objectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateRestOp(), }); const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: objectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateRestOp(), }); await objectsCreatedPromise; @@ -1748,17 +1761,17 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const { objectId: mapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: objectsHelper.mapCreateOp({ - entries: { - foo: { data: { value: 'bar' } }, - baz: { data: { value: 1 } }, + createOp: objectsHelper.mapCreateRestOp({ + data: { + foo: { string: 'bar' }, + baz: { number: 1 }, }, }), }); const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: objectsHelper.counterCreateOp({ count: 1 }), + createOp: objectsHelper.counterCreateRestOp({ number: 1 }), }); await objectsCreatedPromise; @@ -1817,7 +1830,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'foo', - createOp: objectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateRestOp(), }); await objectCreatedPromise; @@ -1858,17 +1871,17 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const { objectId: mapId1 } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map1', - createOp: objectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateRestOp(), }); const { objectId: mapId2 } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map2', - createOp: objectsHelper.mapCreateOp({ entries: { foo: { data: { value: 'bar' } } } }), + createOp: objectsHelper.mapCreateRestOp({ data: { foo: { string: 'bar' } } }), }); const { objectId: counterId1 } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter1', - createOp: objectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateRestOp(), }); await objectsCreatedPromise; @@ -2226,12 +2239,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const keyUpdatedPromise = waitForMapKeyUpdate(root, 'foo'); // send some more operations - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapSetOp({ + objectsHelper.mapSetRestOp({ objectId: 'root', key: 'foo', - data: { value: 'bar' }, + value: { string: 'bar' }, }), ); await keyUpdatedPromise; @@ -2271,7 +2284,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: objectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateRestOp(), }); await counterCreatedPromise; @@ -2314,7 +2327,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: objectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateRestOp(), }); await counterCreatedPromise; @@ -2381,7 +2394,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: objectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateRestOp(), }); await counterCreatedPromise; @@ -2424,7 +2437,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: objectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateRestOp(), }); await counterCreatedPromise; @@ -2529,12 +2542,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: objectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateRestOp(), }); await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: objectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateRestOp(), }); await objectsCreatedPromise; @@ -2569,7 +2582,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: objectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateRestOp(), }); await mapCreatedPromise; @@ -2604,11 +2617,11 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: objectsHelper.mapCreateOp({ - entries: { - foo: { data: { value: 1 } }, - bar: { data: { value: 1 } }, - baz: { data: { value: 1 } }, + createOp: objectsHelper.mapCreateRestOp({ + data: { + foo: { number: 1 }, + bar: { number: 1 }, + baz: { number: 1 }, }, }), }); @@ -2639,7 +2652,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: objectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateRestOp(), }); await mapCreatedPromise; @@ -2886,12 +2899,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'counter', - createOp: objectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateRestOp(), }); await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: 'map', - createOp: objectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateRestOp(), }); await objectsCreatedPromise; @@ -3580,11 +3593,11 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }), ); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.counterIncOp({ + objectsHelper.counterIncRestOp({ objectId: sampleCounterObjectId, - amount: 1, + number: 1, }), ); @@ -3623,11 +3636,11 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ); for (const increment of expectedCounterIncrements) { - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.counterIncOp({ + objectsHelper.counterIncRestOp({ objectId: sampleCounterObjectId, - amount: increment, + number: increment, }), ); } @@ -3657,12 +3670,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }), ); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapSetOp({ + objectsHelper.mapSetRestOp({ objectId: sampleMapObjectId, key: 'stringKey', - data: { value: 'stringValue' }, + value: { string: 'stringValue' }, }), ); @@ -3691,9 +3704,9 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }), ); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapRemoveOp({ + objectsHelper.mapRemoveRestOp({ objectId: sampleMapObjectId, key: 'stringKey', }), @@ -3738,44 +3751,44 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }), ); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapSetOp({ + objectsHelper.mapSetRestOp({ objectId: sampleMapObjectId, key: 'foo', - data: { value: 'something' }, + value: { string: 'something' }, }), ); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapSetOp({ + objectsHelper.mapSetRestOp({ objectId: sampleMapObjectId, key: 'bar', - data: { value: 'something' }, + value: { string: 'something' }, }), ); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapRemoveOp({ + objectsHelper.mapRemoveRestOp({ objectId: sampleMapObjectId, key: 'foo', }), ); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapSetOp({ + objectsHelper.mapSetRestOp({ objectId: sampleMapObjectId, key: 'baz', - data: { value: 'something' }, + value: { string: 'something' }, }), ); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapRemoveOp({ + objectsHelper.mapRemoveRestOp({ objectId: sampleMapObjectId, key: 'bar', }), @@ -3804,11 +3817,11 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const increments = 3; for (let i = 0; i < increments; i++) { const counterUpdatedPromise = waitForCounterUpdate(counter); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.counterIncOp({ + objectsHelper.counterIncRestOp({ objectId: sampleCounterObjectId, - amount: 1, + number: 1, }), ); await counterUpdatedPromise; @@ -3842,11 +3855,11 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const increments = 3; for (let i = 0; i < increments; i++) { const counterUpdatedPromise = waitForCounterUpdate(counter); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.counterIncOp({ + objectsHelper.counterIncRestOp({ objectId: sampleCounterObjectId, - amount: 1, + number: 1, }), ); await counterUpdatedPromise; @@ -3882,11 +3895,11 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const increments = 3; for (let i = 0; i < increments; i++) { const counterUpdatedPromise = waitForCounterUpdate(counter); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.counterIncOp({ + objectsHelper.counterIncRestOp({ objectId: sampleCounterObjectId, - amount: 1, + number: 1, }), ); await counterUpdatedPromise; @@ -3923,12 +3936,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const mapSets = 3; for (let i = 0; i < mapSets; i++) { const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapSetOp({ + objectsHelper.mapSetRestOp({ objectId: sampleMapObjectId, key: `foo-${i}`, - data: { value: 'exists' }, + value: { string: 'exists' }, }), ); await mapUpdatedPromise; @@ -3967,12 +3980,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const mapSets = 3; for (let i = 0; i < mapSets; i++) { const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapSetOp({ + objectsHelper.mapSetRestOp({ objectId: sampleMapObjectId, key: `foo-${i}`, - data: { value: 'exists' }, + value: { string: 'exists' }, }), ); await mapUpdatedPromise; @@ -4013,12 +4026,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const mapSets = 3; for (let i = 0; i < mapSets; i++) { const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapSetOp({ + objectsHelper.mapSetRestOp({ objectId: sampleMapObjectId, key: `foo-${i}`, - data: { value: 'exists' }, + value: { string: 'exists' }, }), ); await mapUpdatedPromise; @@ -4065,12 +4078,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const { objectId: sampleMapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: sampleMapKey, - createOp: objectsHelper.mapCreateOp(), + createOp: objectsHelper.mapCreateRestOp(), }); const { objectId: sampleCounterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { mapObjectId: 'root', key: sampleCounterKey, - createOp: objectsHelper.counterCreateOp(), + createOp: objectsHelper.counterCreateRestOp(), }); await objectsCreatedPromise; @@ -4097,9 +4110,9 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const counterCreatedPromise = waitForObjectOperation(helper, client, ObjectsHelper.ACTIONS.COUNTER_CREATE); // send a CREATE op, this adds an object to the pool - const { objectId } = await objectsHelper.stateRequest( + const { objectId } = await objectsHelper.operationRequest( channelName, - objectsHelper.counterCreateOp({ count: 1 }), + objectsHelper.counterCreateRestOp({ number: 1 }), ); await counterCreatedPromise; @@ -4146,9 +4159,9 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const keyUpdatedPromise = waitForMapKeyUpdate(root, 'foo'); // set a key on a root - await objectsHelper.stateRequest( + await objectsHelper.operationRequest( channelName, - objectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { value: 'bar' } }), + objectsHelper.mapSetRestOp({ objectId: 'root', key: 'foo', value: { string: 'bar' } }), ); await keyUpdatedPromise; @@ -4156,7 +4169,10 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const keyUpdatedPromise2 = waitForMapKeyUpdate(root, 'foo'); // remove the key from the root. this should tombstone the map entry and make it inaccessible to the end user, but still keep it in memory in the underlying map - await objectsHelper.stateRequest(channelName, objectsHelper.mapRemoveOp({ objectId: 'root', key: 'foo' })); + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapRemoveRestOp({ objectId: 'root', key: 'foo' }), + ); await keyUpdatedPromise2; expect(root.get('foo'), 'Check key "foo" is inaccessible via public API on root after MAP_REMOVE').to.not From 4a51ccd3ce671dc587ae39d64a90f93eb3a1da5d Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 16 Apr 2025 10:37:43 +0100 Subject: [PATCH 145/166] `BaseMessage.strMsg` should not print undefined data Encode function refactoring in commit f4073b9b0c8dd19dcd24ee0131690e462f0f7e6f made it so `data` and `encoding` fields will be set on a message object even if they are undefined. This is not a problem in itself, but current `BaseMessage.strMsg` implementation does not check for undefined values and adds them to the result string. It should not do that --- src/common/lib/types/basemessage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/lib/types/basemessage.ts b/src/common/lib/types/basemessage.ts index 00c4788cac..98f8f473ad 100644 --- a/src/common/lib/types/basemessage.ts +++ b/src/common/lib/types/basemessage.ts @@ -370,7 +370,7 @@ export function strMsg(m: any, cls: string) { result += '; data=' + m.data; } else if (Platform.BufferUtils.isBuffer(m.data)) { result += '; data (buffer)=' + Platform.BufferUtils.base64Encode(m.data); - } else { + } else if (typeof m.data !== 'undefined') { result += '; data (json)=' + JSON.stringify(m.data); } } else if (attr && (attr === 'extras' || attr === 'operation')) { From b95ab1fc2fa094ca0a2a4aa5e8f2a42161906831 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 17 Apr 2025 03:09:43 +0100 Subject: [PATCH 146/166] Replace `deep-equal` with `dequal` package deep-equal [1] does not provide UMD-compatible bundle which causes our playwright tests to fail as they cannot resolve the module. Replace it with dequal [2] which has UMD-compatible bundle. (also apparently it should be faster than deep-equal) [1] https://www.npmjs.com/package/deep-equal [2] https://www.npmjs.com/package/dequal --- grunt/esbuild/build.js | 2 +- package-lock.json | 162 +++++++++++++++++----- package.json | 3 +- scripts/moduleReport.ts | 2 +- src/plugins/objects/livemap.ts | 4 +- test/common/globals/named_dependencies.js | 2 +- 6 files changed, 137 insertions(+), 38 deletions(-) diff --git a/grunt/esbuild/build.js b/grunt/esbuild/build.js index ec809f4bde..2144ce2709 100644 --- a/grunt/esbuild/build.js +++ b/grunt/esbuild/build.js @@ -82,7 +82,7 @@ const objectsPluginConfig = { entryPoints: ['src/plugins/objects/index.ts'], plugins: [umdWrapper.default({ libraryName: 'AblyObjectsPlugin', amdNamedModule: false })], outfile: 'build/objects.js', - external: ['deep-equal'], + external: ['dequal'], }; const objectsPluginCdnConfig = { diff --git a/package-lock.json b/package-lock.json index e90770e81d..d09be29865 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@ably/msgpack-js": "^0.4.0", - "deep-equal": "^2.2.3", + "dequal": "^2.0.3", "fastestsmallesttextencoderdecoder": "^1.0.22", "got": "^11.8.5", "ulid": "^2.3.0", @@ -24,7 +24,6 @@ "@babel/traverse": "^7.23.7", "@testing-library/react": "^13.3.0", "@types/cli-table": "^0.3.4", - "@types/deep-equal": "^1.0.4", "@types/jmespath": "^0.15.2", "@types/node": "^18.0.0", "@types/request": "^2.48.7", @@ -1559,12 +1558,6 @@ "integrity": "sha512-GsALrTL69mlwbAw/MHF1IPTadSLZQnsxe7a80G8l4inN/iEXCOcVeT/S7aRc6hbhqzL9qZ314kHPDQnQ3ev+HA==", "dev": true }, - "node_modules/@types/deep-equal": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@types/deep-equal/-/deep-equal-1.0.4.tgz", - "integrity": "sha512-tqdiS4otQP4KmY0PR3u6KbZ5EWvhNdUoS/jc93UuK23C220lOZ/9TvjfxdPcKvqwwDVtmtSCrnr0p/2dirAxkA==", - "dev": true - }, "node_modules/@types/eslint": { "version": "8.56.2", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.2.tgz", @@ -2328,6 +2321,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz", "integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==", + "dev": true, "dependencies": { "call-bind": "^1.0.2", "is-array-buffer": "^3.0.1" @@ -2520,6 +2514,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", + "dev": true, "engines": { "node": ">= 0.4" }, @@ -2755,6 +2750,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", + "dev": true, "dependencies": { "function-bind": "^1.1.2", "get-intrinsic": "^1.2.1", @@ -3293,6 +3289,7 @@ "version": "2.2.3", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.3.tgz", "integrity": "sha512-ZIwpnevOurS8bpT4192sqAowWM76JDKSHYzMLty3BZGSswgq6pBaH3DhCSW5xVAZICZyKdOBPjwww5wfgT/6PA==", + "dev": true, "dependencies": { "array-buffer-byte-length": "^1.0.0", "call-bind": "^1.0.5", @@ -3323,7 +3320,8 @@ "node_modules/deep-equal/node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "dev": true }, "node_modules/deep-for-each": { "version": "3.0.0", @@ -3352,6 +3350,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", + "dev": true, "dependencies": { "get-intrinsic": "^1.2.1", "gopd": "^1.0.1", @@ -3365,6 +3364,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "dev": true, "dependencies": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", @@ -3395,6 +3395,15 @@ "node": ">= 0.8" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/destroy": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", @@ -3664,6 +3673,7 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz", "integrity": "sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw==", + "dev": true, "dependencies": { "call-bind": "^1.0.2", "get-intrinsic": "^1.1.3", @@ -3682,7 +3692,8 @@ "node_modules/es-get-iterator/node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "dev": true }, "node_modules/es-iterator-helpers": { "version": "1.0.15", @@ -4777,7 +4788,8 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/fast-glob": { "version": "3.3.2", @@ -5012,6 +5024,7 @@ "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "dev": true, "dependencies": { "is-callable": "^1.1.3" } @@ -5109,6 +5122,7 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -5141,6 +5155,7 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", + "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -5176,6 +5191,7 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", + "dev": true, "dependencies": { "function-bind": "^1.1.2", "has-proto": "^1.0.1", @@ -5380,6 +5396,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dev": true, "dependencies": { "get-intrinsic": "^1.1.3" }, @@ -5786,6 +5803,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", + "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -5803,6 +5821,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz", "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==", + "dev": true, "dependencies": { "get-intrinsic": "^1.2.2" }, @@ -5814,6 +5833,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "dev": true, "engines": { "node": ">= 0.4" }, @@ -5825,6 +5845,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true, "engines": { "node": ">= 0.4" }, @@ -5836,6 +5857,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "dev": true, "dependencies": { "has-symbols": "^1.0.2" }, @@ -5850,6 +5872,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "dev": true, "dependencies": { "function-bind": "^1.1.2" }, @@ -6065,6 +6088,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.6.tgz", "integrity": "sha512-Xj6dv+PsbtwyPpEflsejS+oIZxmMlV44zAhG479uYu89MsjcYOhCFnNyKrkJrihbsiasQyY0afoCl/9BLR65bg==", + "dev": true, "dependencies": { "get-intrinsic": "^1.2.2", "hasown": "^2.0.0", @@ -6109,6 +6133,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", + "dev": true, "dependencies": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -6124,6 +6149,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz", "integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==", + "dev": true, "dependencies": { "call-bind": "^1.0.2", "get-intrinsic": "^1.2.0", @@ -6152,6 +6178,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "dev": true, "dependencies": { "has-bigints": "^1.0.1" }, @@ -6175,6 +6202,7 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "dev": true, "dependencies": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -6196,6 +6224,7 @@ "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "dev": true, "engines": { "node": ">= 0.4" }, @@ -6219,6 +6248,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "dev": true, "dependencies": { "has-tostringtag": "^1.0.0" }, @@ -6305,6 +6335,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.2.tgz", "integrity": "sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==", + "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -6334,6 +6365,7 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "dev": true, "dependencies": { "has-tostringtag": "^1.0.0" }, @@ -6375,6 +6407,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "dev": true, "dependencies": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -6402,6 +6435,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.2.tgz", "integrity": "sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==", + "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -6410,6 +6444,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", + "dev": true, "dependencies": { "call-bind": "^1.0.2" }, @@ -6421,6 +6456,7 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "dev": true, "dependencies": { "has-tostringtag": "^1.0.0" }, @@ -6435,6 +6471,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "dev": true, "dependencies": { "has-symbols": "^1.0.2" }, @@ -6449,6 +6486,7 @@ "version": "1.1.12", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", + "dev": true, "dependencies": { "which-typed-array": "^1.1.11" }, @@ -6475,6 +6513,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz", "integrity": "sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==", + "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -6495,6 +6534,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.2.tgz", "integrity": "sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg==", + "dev": true, "dependencies": { "call-bind": "^1.0.2", "get-intrinsic": "^1.1.1" @@ -7561,6 +7601,7 @@ "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", + "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -7569,6 +7610,7 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz", "integrity": "sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==", + "dev": true, "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.3" @@ -7584,6 +7626,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true, "engines": { "node": ">= 0.4" } @@ -7592,6 +7635,7 @@ "version": "4.1.5", "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz", "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==", + "dev": true, "dependencies": { "call-bind": "^1.0.5", "define-properties": "^1.2.1", @@ -8532,6 +8576,7 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz", "integrity": "sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg==", + "dev": true, "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.2.0", @@ -8961,6 +9006,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.0.tgz", "integrity": "sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w==", + "dev": true, "dependencies": { "define-data-property": "^1.1.1", "function-bind": "^1.1.2", @@ -8976,6 +9022,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.1.tgz", "integrity": "sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==", + "dev": true, "dependencies": { "define-data-property": "^1.0.1", "functions-have-names": "^1.2.3", @@ -9077,6 +9124,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dev": true, "dependencies": { "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", @@ -9267,6 +9315,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz", "integrity": "sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==", + "dev": true, "dependencies": { "internal-slot": "^1.0.4" }, @@ -10827,6 +10876,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "dev": true, "dependencies": { "is-bigint": "^1.0.1", "is-boolean-object": "^1.1.0", @@ -10874,6 +10924,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.1.tgz", "integrity": "sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==", + "dev": true, "dependencies": { "is-map": "^2.0.1", "is-set": "^2.0.1", @@ -10888,6 +10939,7 @@ "version": "1.1.13", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.13.tgz", "integrity": "sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==", + "dev": true, "dependencies": { "available-typed-arrays": "^1.0.5", "call-bind": "^1.0.4", @@ -12130,12 +12182,6 @@ "integrity": "sha512-GsALrTL69mlwbAw/MHF1IPTadSLZQnsxe7a80G8l4inN/iEXCOcVeT/S7aRc6hbhqzL9qZ314kHPDQnQ3ev+HA==", "dev": true }, - "@types/deep-equal": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@types/deep-equal/-/deep-equal-1.0.4.tgz", - "integrity": "sha512-tqdiS4otQP4KmY0PR3u6KbZ5EWvhNdUoS/jc93UuK23C220lOZ/9TvjfxdPcKvqwwDVtmtSCrnr0p/2dirAxkA==", - "dev": true - }, "@types/eslint": { "version": "8.56.2", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.2.tgz", @@ -12735,6 +12781,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz", "integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==", + "dev": true, "requires": { "call-bind": "^1.0.2", "is-array-buffer": "^3.0.1" @@ -12877,7 +12924,8 @@ "available-typed-arrays": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", - "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" + "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", + "dev": true }, "aws-sdk": { "version": "2.1539.0", @@ -13058,6 +13106,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", + "dev": true, "requires": { "function-bind": "^1.1.2", "get-intrinsic": "^1.2.1", @@ -13456,6 +13505,7 @@ "version": "2.2.3", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.3.tgz", "integrity": "sha512-ZIwpnevOurS8bpT4192sqAowWM76JDKSHYzMLty3BZGSswgq6pBaH3DhCSW5xVAZICZyKdOBPjwww5wfgT/6PA==", + "dev": true, "requires": { "array-buffer-byte-length": "^1.0.0", "call-bind": "^1.0.5", @@ -13480,7 +13530,8 @@ "isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "dev": true } } }, @@ -13508,6 +13559,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", + "dev": true, "requires": { "get-intrinsic": "^1.2.1", "gopd": "^1.0.1", @@ -13518,6 +13570,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "dev": true, "requires": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", @@ -13536,6 +13589,11 @@ "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "dev": true }, + "dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==" + }, "destroy": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", @@ -13748,6 +13806,7 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz", "integrity": "sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw==", + "dev": true, "requires": { "call-bind": "^1.0.2", "get-intrinsic": "^1.1.3", @@ -13763,7 +13822,8 @@ "isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "dev": true } } }, @@ -14721,6 +14781,7 @@ "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "dev": true, "requires": { "is-callable": "^1.1.3" } @@ -14789,7 +14850,8 @@ "function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true }, "function.prototype.name": { "version": "1.1.6", @@ -14812,7 +14874,8 @@ "functions-have-names": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", - "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==" + "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", + "dev": true }, "gensync": { "version": "1.0.0-beta.2", @@ -14836,6 +14899,7 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", + "dev": true, "requires": { "function-bind": "^1.1.2", "has-proto": "^1.0.1", @@ -14984,6 +15048,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dev": true, "requires": { "get-intrinsic": "^1.1.3" } @@ -15297,7 +15362,8 @@ "has-bigints": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", - "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==" + "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", + "dev": true }, "has-flag": { "version": "4.0.0", @@ -15309,6 +15375,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz", "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==", + "dev": true, "requires": { "get-intrinsic": "^1.2.2" } @@ -15316,17 +15383,20 @@ "has-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", - "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==" + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "dev": true }, "has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true }, "has-tostringtag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "dev": true, "requires": { "has-symbols": "^1.0.2" } @@ -15335,6 +15405,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "dev": true, "requires": { "function-bind": "^1.1.2" } @@ -15496,6 +15567,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.6.tgz", "integrity": "sha512-Xj6dv+PsbtwyPpEflsejS+oIZxmMlV44zAhG479uYu89MsjcYOhCFnNyKrkJrihbsiasQyY0afoCl/9BLR65bg==", + "dev": true, "requires": { "get-intrinsic": "^1.2.2", "hasown": "^2.0.0", @@ -15528,6 +15600,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", + "dev": true, "requires": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -15537,6 +15610,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz", "integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==", + "dev": true, "requires": { "call-bind": "^1.0.2", "get-intrinsic": "^1.2.0", @@ -15556,6 +15630,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "dev": true, "requires": { "has-bigints": "^1.0.1" } @@ -15573,6 +15648,7 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "dev": true, "requires": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -15587,7 +15663,8 @@ "is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==" + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "dev": true }, "is-core-module": { "version": "2.13.1", @@ -15602,6 +15679,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "dev": true, "requires": { "has-tostringtag": "^1.0.0" } @@ -15654,7 +15732,8 @@ "is-map": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.2.tgz", - "integrity": "sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==" + "integrity": "sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==", + "dev": true }, "is-negative-zero": { "version": "2.0.2", @@ -15672,6 +15751,7 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "dev": true, "requires": { "has-tostringtag": "^1.0.0" } @@ -15701,6 +15781,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "dev": true, "requires": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -15718,12 +15799,14 @@ "is-set": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.2.tgz", - "integrity": "sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==" + "integrity": "sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==", + "dev": true }, "is-shared-array-buffer": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", + "dev": true, "requires": { "call-bind": "^1.0.2" } @@ -15732,6 +15815,7 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "dev": true, "requires": { "has-tostringtag": "^1.0.0" } @@ -15740,6 +15824,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "dev": true, "requires": { "has-symbols": "^1.0.2" } @@ -15748,6 +15833,7 @@ "version": "1.1.12", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", + "dev": true, "requires": { "which-typed-array": "^1.1.11" } @@ -15764,7 +15850,8 @@ "is-weakmap": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz", - "integrity": "sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==" + "integrity": "sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==", + "dev": true }, "is-weakref": { "version": "1.0.2", @@ -15779,6 +15866,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.2.tgz", "integrity": "sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg==", + "dev": true, "requires": { "call-bind": "^1.0.2", "get-intrinsic": "^1.1.1" @@ -16585,12 +16673,14 @@ "object-inspect": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", - "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==" + "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", + "dev": true }, "object-is": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz", "integrity": "sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==", + "dev": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3" @@ -16599,12 +16689,14 @@ "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true }, "object.assign": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz", "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==", + "dev": true, "requires": { "call-bind": "^1.0.5", "define-properties": "^1.2.1", @@ -17273,6 +17365,7 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz", "integrity": "sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg==", + "dev": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.2.0", @@ -17588,6 +17681,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.0.tgz", "integrity": "sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w==", + "dev": true, "requires": { "define-data-property": "^1.1.1", "function-bind": "^1.1.2", @@ -17600,6 +17694,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.1.tgz", "integrity": "sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==", + "dev": true, "requires": { "define-data-property": "^1.0.1", "functions-have-names": "^1.2.3", @@ -17679,6 +17774,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dev": true, "requires": { "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", @@ -17827,6 +17923,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz", "integrity": "sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==", + "dev": true, "requires": { "internal-slot": "^1.0.4" } @@ -18897,6 +18994,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "dev": true, "requires": { "is-bigint": "^1.0.1", "is-boolean-object": "^1.1.0", @@ -18937,6 +19035,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.1.tgz", "integrity": "sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==", + "dev": true, "requires": { "is-map": "^2.0.1", "is-set": "^2.0.1", @@ -18948,6 +19047,7 @@ "version": "1.1.13", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.13.tgz", "integrity": "sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==", + "dev": true, "requires": { "available-typed-arrays": "^1.0.5", "call-bind": "^1.0.4", diff --git a/package.json b/package.json index fc60927ead..b4cc4ebb02 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ ], "dependencies": { "@ably/msgpack-js": "^0.4.0", - "deep-equal": "^2.2.3", + "dequal": "^2.0.3", "fastestsmallesttextencoderdecoder": "^1.0.22", "got": "^11.8.5", "ulid": "^2.3.0", @@ -73,7 +73,6 @@ "@babel/traverse": "^7.23.7", "@testing-library/react": "^13.3.0", "@types/cli-table": "^0.3.4", - "@types/deep-equal": "^1.0.4", "@types/jmespath": "^0.15.2", "@types/node": "^18.0.0", "@types/request": "^2.48.7", diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index 2c09047b57..ef7e6ec8bb 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -48,7 +48,7 @@ interface PluginInfo { const buildablePlugins: Record<'push' | 'objects', PluginInfo> = { push: { description: 'Push', path: './build/push.js', external: ['ulid'] }, - objects: { description: 'Objects', path: './build/objects.js', external: ['deep-equal'] }, + objects: { description: 'Objects', path: './build/objects.js', external: ['dequal'] }, }; function formatBytes(bytes: number) { diff --git a/src/plugins/objects/livemap.ts b/src/plugins/objects/livemap.ts index 90f4d7d023..22eaa0a29e 100644 --- a/src/plugins/objects/livemap.ts +++ b/src/plugins/objects/livemap.ts @@ -1,4 +1,4 @@ -import deepEqual from 'deep-equal'; +import { dequal } from 'dequal'; import type * as API from '../../../ably'; import { DEFAULTS } from './defaults'; @@ -572,7 +572,7 @@ export class LiveMap extends LiveObject Date: Thu, 17 Apr 2025 06:27:54 +0100 Subject: [PATCH 147/166] Update LiveObjects docs link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fc4401bb32..a104ddb18c 100644 --- a/README.md +++ b/README.md @@ -621,7 +621,7 @@ const client = new Ably.Realtime({ The Objects plugin is developed as part of the Ably client library, so it is available for the same versions as the Ably client library itself. It also means that it follows the same semantic versioning rules as they were defined for [the Ably client library](#for-browsers). For example, to lock into a major or minor version of the Objects plugin, you can specify a specific version number such as https://cdn.ably.com/lib/objects.umd.min-2.js for all v2._ versions, or https://cdn.ably.com/lib/objects.umd.min-2.4.js for all v2.4._ versions, or you can lock into a single release with https://cdn.ably.com/lib/objects.umd.min-2.4.0.js. Note you can load the non-minified version by omitting `.min` from the URL such as https://cdn.ably.com/lib/objects.umd-2.js. -For more information about the LiveObjects product, see the [Ably LiveObjects documentation](https://ably.com/docs/products/liveobjects). +For more information about the LiveObjects product, see the [Ably LiveObjects documentation](https://ably.com/docs/liveobjects). #### Objects Channel Modes From 335e8086404f9e557fd14a3189b0502d85e7cf2b Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 28 Apr 2025 14:16:48 +0100 Subject: [PATCH 148/166] Update test for RTL4c1 to not set `channelSerial` externally --- test/common/modules/private_api_recorder.js | 2 + test/realtime/channel.test.js | 69 +++++++++++++++------ 2 files changed, 52 insertions(+), 19 deletions(-) diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index 986e8d8dc7..8aec585500 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -82,6 +82,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'read.LiveMap._dataRef.data', 'read.EventEmitter.events', 'read.Platform.Config.push', + 'read.ProtocolMessage.channelSerial', 'read.Realtime._transports', 'read.auth.authOptions.authUrl', 'read.auth.key', @@ -89,6 +90,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'read.auth.tokenParams.version', 'read.channel.channelOptions', 'read.channel.channelOptions.cipher', + 'read.channel.properties.channelSerial', // This should be public API, but channel.properties is not currently exposed. Remove it from the list when https://github.com/ably/ably-js/issues/2018 is done 'read.connectionManager.activeProtocol', 'read.connectionManager.activeProtocol.transport', 'read.connectionManager.baseTransport', diff --git a/test/realtime/channel.test.js b/test/realtime/channel.test.js index e4beaab8ef..f791409605 100644 --- a/test/realtime/channel.test.js +++ b/test/realtime/channel.test.js @@ -1872,34 +1872,63 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async const realtime = helper.AblyRealtime(); await helper.monitorConnectionAsync(async () => { - const channel = realtime.channels.get('channel'); - channel.properties.channelSerial = 'channelSerial'; - + const channel = realtime.channels.get('set_channelSerial_on_attach'); await realtime.connection.once('connected'); + await channel.attach(); - const promiseCheck = new Promise((resolve, reject) => { + // Publish a message to get the channelSerial from it + const messageReceivedPromise = new Promise((resolve, reject) => { helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); const transport = realtime.connection.connectionManager.activeProtocol.getTransport(); - const sendOriginal = transport.send; + const onProtocolMessageOriginal = transport.onProtocolMessage; - helper.recordPrivateApi('replace.transport.send'); - transport.send = function (msg) { - if (msg.action === 10) { - try { - expect(msg.channelSerial).to.equal('channelSerial'); - resolve(); - } catch (error) { - reject(error); + helper.recordPrivateApi('replace.transport.onProtocolMessage'); + transport.onProtocolMessage = function (msg) { + if (msg.action === 15) { + // MESSAGE + resolve(msg); + } + + helper.recordPrivateApi('call.transport.onProtocolMessage'); + onProtocolMessageOriginal.call(this, msg); + }; + }); + await channel.publish('event', 'test'); + + const receivedMessage = await messageReceivedPromise; + helper.recordPrivateApi('read.ProtocolMessage.channelSerial'); + const receivedChannelSerial = receivedMessage.channelSerial; + + // After the disconnect, on reconnect, spy on transport.send to check sent channelSerial + const promiseCheck = new Promise((resolve, reject) => { + helper.recordPrivateApi('listen.connectionManager.transport.pending'); + realtime.connection.connectionManager.once('transport.pending', function (transport) { + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + const sendOriginal = transport.send; + + helper.recordPrivateApi('replace.transport.send'); + transport.send = function (msg) { + if (msg.action === 10) { + // ATTACH + try { + helper.recordPrivateApi('read.ProtocolMessage.channelSerial'); + expect(msg.channelSerial).to.equal(receivedChannelSerial); + resolve(); + } catch (error) { + reject(error); + } } - } else { + helper.recordPrivateApi('call.transport.send'); sendOriginal.call(this, msg); - } - }; + }; + }); }); - // don't await for attach as it will never resolve in this test since we don't send ATTACH msg to realtime - channel.attach(); + // Disconnect the transport (will automatically reconnect and resume) + helper.recordPrivateApi('call.connectionManager.disconnectAllTransports'); + realtime.connection.connectionManager.disconnectAllTransports(); + await promiseCheck; }, realtime); @@ -1912,7 +1941,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async const realtime = helper.AblyRealtime({ clientId: 'me' }); await helper.monitorConnectionAsync(async () => { - const channel = realtime.channels.get('channel'); + const channel = realtime.channels.get('update_channelSerial_on_message'); await realtime.connection.once('connected'); helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); @@ -1949,6 +1978,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async // wait until next event loop so any async ops get resolved and channel serial gets updated on a channel await new Promise((res) => setTimeout(res, 0)); + helper.recordPrivateApi('read.channel.properties.channelSerial'); + helper.recordPrivateApi('read.ProtocolMessage.channelSerial'); expect(channel.properties.channelSerial).to.equal(msg.channelSerial); } }, realtime); From bc8d2cf4a152e6ebe023a13460c9b69d6e4d11ec Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 28 Apr 2025 14:23:39 +0100 Subject: [PATCH 149/166] Add `ANNOTATION` to RTL15b test --- test/realtime/channel.test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/realtime/channel.test.js b/test/realtime/channel.test.js index f791409605..2f3280e200 100644 --- a/test/realtime/channel.test.js +++ b/test/realtime/channel.test.js @@ -1967,6 +1967,11 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channel: channel.name, channelSerial: 'OBJECT', }), + createPM({ + action: 21, // ANNOTATION + channel: channel.name, + channelSerial: 'ANNOTATION', + }), ]; helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); From 5599ad8723c9920d97d82f6f85df84a6f65f2db0 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 28 Apr 2025 14:31:04 +0100 Subject: [PATCH 150/166] Add comment for Utils.dataSizeBytes to explain its return values --- src/common/lib/util/utils.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/common/lib/util/utils.ts b/src/common/lib/util/utils.ts index 7883236c94..351daa96b5 100644 --- a/src/common/lib/util/utils.ts +++ b/src/common/lib/util/utils.ts @@ -279,7 +279,15 @@ export function inspectBody(body: unknown): string { } } -/* Data is assumed to be either a string, a number, a boolean or a buffer. */ +/** + * Data is assumed to be either a string, a number, a boolean or a buffer. + * + * Returns the byte size of the provided data based on the spec: + * - TM6a - size of the string is byte length of the string + * - TM6c - size of the buffer is its size in bytes + * - OD3c - size of a number is 8 bytes + * - OD3d - size of a boolean is 1 byte + */ export function dataSizeBytes(data: string | number | boolean | Bufferlike): number { if (Platform.BufferUtils.isBuffer(data)) { return Platform.BufferUtils.byteLength(data); From 95be48c0807ce02c82bdf4651213b18842044316 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 28 Apr 2025 14:34:26 +0100 Subject: [PATCH 151/166] Update spelling in ably.d.ts --- ably.d.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ably.d.ts b/ably.d.ts index 3ad725ca63..efab4ca2a1 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -880,7 +880,7 @@ declare namespace ChannelModes { */ type OBJECT_PUBLISH = 'OBJECT_PUBLISH' | 'object_publish'; /** - * The client can receive object messages. + * The client will receive object messages. */ type OBJECT_SUBSCRIBE = 'OBJECT_SUBSCRIBE' | 'object_subscribe'; /** @@ -933,15 +933,15 @@ declare namespace ResolvedChannelModes { */ type OBJECT_PUBLISH = 'object_publish'; /** - * The client can receive object messages. + * The client will receive object messages. */ type OBJECT_SUBSCRIBE = 'object_subscribe'; /** - * The client can publish annotations + * The client can publish annotations. */ type ANNOTATION_PUBLISH = 'annotation_publish'; /** - * The client will receive annotations + * The client will receive annotations. */ type ANNOTATION_SUBSCRIBE = 'annotation_subscribe'; } From bd3873c3f17864c0b74c59025430c74f7f64e9ee Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 28 Apr 2025 14:37:25 +0100 Subject: [PATCH 152/166] Rename `ObjectsTypes` to `AblyObjectsTypes` --- README.md | 6 ++--- ably.d.ts | 24 +++++++++---------- .../browser/template/src/ably.config.d.ts | 2 +- .../browser/template/src/index-objects.ts | 8 +++---- typedoc.json | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 0bdae9e0e9..b566b683bf 100644 --- a/README.md +++ b/README.md @@ -866,7 +866,7 @@ objects.offAll(); #### Typing Objects -You can provide your own TypeScript typings for Objects by providing a globally defined `ObjectsTypes` interface. +You can provide your own TypeScript typings for Objects by providing a globally defined `AblyObjectsTypes` interface. ```typescript // file: ably.config.d.ts @@ -880,7 +880,7 @@ type MyCustomRoot = { }; declare global { - export interface ObjectsTypes { + export interface AblyObjectsTypes { root: MyCustomRoot; } } @@ -889,7 +889,7 @@ declare global { This will enable code completion and editor hints when interacting with the Objects API: ```typescript -const root = await objects.getRoot(); // uses types defined by global ObjectsTypes interface by default +const root = await objects.getRoot(); // uses types defined by global AblyObjectsTypes interface by default const map = root.get('map'); // LiveMap<{ foo: string; counter: LiveCounter }> map.set('foo', 1); // TypeError diff --git a/ably.d.ts b/ably.d.ts index efab4ca2a1..f2f5301b32 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -2277,9 +2277,9 @@ export declare interface Objects { /** * Retrieves the root {@link LiveMap} object for Objects on a channel. * - * A type parameter can be provided to describe the structure of the Objects on the channel. By default, it uses types from the globally defined `ObjectsTypes` interface. + * A type parameter can be provided to describe the structure of the Objects on the channel. By default, it uses types from the globally defined `AblyObjectsTypes` interface. * - * You can specify custom types for Objects by defining a global `ObjectsTypes` interface with a `root` property that conforms to {@link LiveMapType}. + * You can specify custom types for Objects by defining a global `AblyObjectsTypes` interface with a `root` property that conforms to {@link LiveMapType}. * * Example: * @@ -2291,7 +2291,7 @@ export declare interface Objects { * }; * * declare global { - * export interface ObjectsTypes { + * export interface AblyObjectsTypes { * root: MyRoot; * } * } @@ -2367,7 +2367,7 @@ declare global { /** * A globally defined interface that allows users to define custom types for Objects. */ - export interface ObjectsTypes { + export interface AblyObjectsTypes { [key: string]: unknown; } } @@ -2379,20 +2379,20 @@ declare global { export type LiveMapType = { [key: string]: ObjectValue | LiveMap | LiveCounter | undefined }; /** - * The default type for the `root` object for Objects on a channel, based on the globally defined {@link ObjectsTypes} interface. + * The default type for the `root` object for Objects on a channel, based on the globally defined {@link AblyObjectsTypes} interface. * - * - If no custom types are provided in `ObjectsTypes`, defaults to an untyped root map representation using the {@link LiveMapType} interface. - * - If a `root` type exists in `ObjectsTypes` and conforms to the {@link LiveMapType} interface, it is used as the type for the `root` object. + * - If no custom types are provided in `AblyObjectsTypes`, defaults to an untyped root map representation using the {@link LiveMapType} interface. + * - If a `root` type exists in `AblyObjectsTypes` and conforms to the {@link LiveMapType} interface, it is used as the type for the `root` object. * - If the provided `root` type does not match {@link LiveMapType}, a type error message is returned. */ export type DefaultRoot = // we need a way to know when no types were provided by the user. - // we expect a "root" property to be set on ObjectsTypes interface, e.g. it won't be "unknown" anymore - unknown extends ObjectsTypes['root'] + // we expect a "root" property to be set on AblyObjectsTypes interface, e.g. it won't be "unknown" anymore + unknown extends AblyObjectsTypes['root'] ? LiveMapType // no custom types provided; use the default untyped map representation for the root - : ObjectsTypes['root'] extends LiveMapType - ? ObjectsTypes['root'] // "root" property exists, and it is of an expected type, we can use this interface for the root object in Objects. - : `Provided type definition for the "root" object in ObjectsTypes is not of an expected LiveMapType`; + : AblyObjectsTypes['root'] extends LiveMapType + ? AblyObjectsTypes['root'] // "root" property exists, and it is of an expected type, we can use this interface for the root object in Objects. + : `Provided type definition for the "root" object in AblyObjectsTypes is not of an expected LiveMapType`; /** * Object returned from an `on` call, allowing the listener provided in that call to be deregistered. diff --git a/test/package/browser/template/src/ably.config.d.ts b/test/package/browser/template/src/ably.config.d.ts index a9e596d3ab..c32f3277d4 100644 --- a/test/package/browser/template/src/ably.config.d.ts +++ b/test/package/browser/template/src/ably.config.d.ts @@ -15,7 +15,7 @@ type CustomRoot = { }; declare global { - export interface ObjectsTypes { + export interface AblyObjectsTypes { root: CustomRoot; } } diff --git a/test/package/browser/template/src/index-objects.ts b/test/package/browser/template/src/index-objects.ts index 3796c529e2..509e780982 100644 --- a/test/package/browser/template/src/index-objects.ts +++ b/test/package/browser/template/src/index-objects.ts @@ -16,14 +16,14 @@ globalThis.testAblyPackage = async function () { // check Objects can be accessed const objects = channel.objects; await channel.attach(); - // expect root to be a LiveMap instance with Objects types defined via the global ObjectsTypes interface + // expect root to be a LiveMap instance with Objects types defined via the global AblyObjectsTypes interface // also checks that we can refer to the Objects types exported from 'ably' by referencing a LiveMap interface const root: Ably.LiveMap = await objects.getRoot(); // check root has expected LiveMap TypeScript type methods const size: number = root.size(); - // check custom user provided typings via ObjectsTypes are working: + // check custom user provided typings via AblyObjectsTypes are working: // any LiveMap.get() call can return undefined, as the LiveMap itself can be tombstoned (has empty state), // or referenced object is tombstoned. // keys on a root: @@ -33,7 +33,7 @@ globalThis.testAblyPackage = async function () { const userProvidedUndefined: string | undefined = root.get('couldBeUndefined'); // objects on a root: const counter: Ably.LiveCounter | undefined = root.get('counterKey'); - const map: ObjectsTypes['root']['mapKey'] | undefined = root.get('mapKey'); + const map: AblyObjectsTypes['root']['mapKey'] | undefined = root.get('mapKey'); // check string literal types works // need to use nullish coalescing as we didn't actually create any data on the root, // so the next calls would fail. we only need to check that TypeScript types work @@ -64,7 +64,7 @@ globalThis.testAblyPackage = async function () { }); counterSubscribeResponse?.unsubscribe(); - // check can provide custom types for the getRoot method, ignoring global ObjectsTypes interface + // check can provide custom types for the getRoot method, ignoring global AblyObjectsTypes interface const explicitRoot: Ably.LiveMap = await objects.getRoot(); const someOtherKey: string | undefined = explicitRoot.get('someOtherKey'); }; diff --git a/typedoc.json b/typedoc.json index 4474605cda..bdbe58b4dc 100644 --- a/typedoc.json +++ b/typedoc.json @@ -21,5 +21,5 @@ "Variable", "Namespace" ], - "intentionallyNotExported": ["__global.ObjectsTypes"] + "intentionallyNotExported": ["__global.AblyObjectsTypes"] } From 0643266a6cc5c8b6a59ab4531e1a03696cb616e0 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 28 Apr 2025 14:46:23 +0100 Subject: [PATCH 153/166] Use `"strict": true` in package tests --- test/package/browser/template/src/index-default.ts | 6 ++++++ test/package/browser/template/src/index-modular.ts | 6 ++++++ test/package/browser/template/src/index-objects.ts | 6 ++++++ test/package/browser/template/src/tsconfig.json | 2 +- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/test/package/browser/template/src/index-default.ts b/test/package/browser/template/src/index-default.ts index cda822d21b..862d0bd9bc 100644 --- a/test/package/browser/template/src/index-default.ts +++ b/test/package/browser/template/src/index-default.ts @@ -1,6 +1,12 @@ import * as Ably from 'ably'; import { createSandboxAblyAPIKey } from './sandbox'; +// Fix for "type 'typeof globalThis' has no index signature" error: +// https://stackoverflow.com/questions/68481686/type-typeof-globalthis-has-no-index-signature +declare module globalThis { + var testAblyPackage: () => Promise; +} + // This function exists to check that we can refer to the types exported by Ably. async function attachChannel(channel: Ably.RealtimeChannel) { await channel.attach(); diff --git a/test/package/browser/template/src/index-modular.ts b/test/package/browser/template/src/index-modular.ts index a9186f56dd..46b06e4617 100644 --- a/test/package/browser/template/src/index-modular.ts +++ b/test/package/browser/template/src/index-modular.ts @@ -2,6 +2,12 @@ import { BaseRealtime, WebSocketTransport, FetchRequest, generateRandomKey } fro import { InboundMessage, RealtimeChannel } from 'ably'; import { createSandboxAblyAPIKey } from './sandbox'; +// Fix for "type 'typeof globalThis' has no index signature" error: +// https://stackoverflow.com/questions/68481686/type-typeof-globalthis-has-no-index-signature +declare module globalThis { + var testAblyPackage: () => Promise; +} + // This function exists to check that we can refer to the types exported by Ably. async function attachChannel(channel: RealtimeChannel) { await channel.attach(); diff --git a/test/package/browser/template/src/index-objects.ts b/test/package/browser/template/src/index-objects.ts index 509e780982..b6f567cf55 100644 --- a/test/package/browser/template/src/index-objects.ts +++ b/test/package/browser/template/src/index-objects.ts @@ -3,6 +3,12 @@ import Objects from 'ably/objects'; import { CustomRoot } from './ably.config'; import { createSandboxAblyAPIKey } from './sandbox'; +// Fix for "type 'typeof globalThis' has no index signature" error: +// https://stackoverflow.com/questions/68481686/type-typeof-globalthis-has-no-index-signature +declare module globalThis { + var testAblyPackage: () => Promise; +} + type ExplicitRootType = { someOtherKey: string; }; diff --git a/test/package/browser/template/src/tsconfig.json b/test/package/browser/template/src/tsconfig.json index 3230e8697f..925607050a 100644 --- a/test/package/browser/template/src/tsconfig.json +++ b/test/package/browser/template/src/tsconfig.json @@ -1,7 +1,7 @@ { "include": ["**/*.ts", "**/*.tsx"], "compilerOptions": { - "strictNullChecks": true, + "strict": true, "resolveJsonModule": true, "esModuleInterop": true, "module": "esnext", From 73bcc1daec2dfb1c19f125e8ae4f6af62073457c Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 28 Apr 2025 14:49:30 +0100 Subject: [PATCH 154/166] Move `AblyObjectsTypes` definition to `index-objects` file in package test --- .../browser/template/src/ably.config.d.ts | 21 ------------------- .../browser/template/src/index-objects.ts | 21 ++++++++++++++++++- 2 files changed, 20 insertions(+), 22 deletions(-) delete mode 100644 test/package/browser/template/src/ably.config.d.ts diff --git a/test/package/browser/template/src/ably.config.d.ts b/test/package/browser/template/src/ably.config.d.ts deleted file mode 100644 index c32f3277d4..0000000000 --- a/test/package/browser/template/src/ably.config.d.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { LiveCounter, LiveMap } from 'ably'; - -type CustomRoot = { - numberKey: number; - stringKey: string; - booleanKey: boolean; - couldBeUndefined?: string; - mapKey: LiveMap<{ - foo: 'bar'; - nestedMap?: LiveMap<{ - baz: 'qux'; - }>; - }>; - counterKey: LiveCounter; -}; - -declare global { - export interface AblyObjectsTypes { - root: CustomRoot; - } -} diff --git a/test/package/browser/template/src/index-objects.ts b/test/package/browser/template/src/index-objects.ts index b6f567cf55..7dbca53573 100644 --- a/test/package/browser/template/src/index-objects.ts +++ b/test/package/browser/template/src/index-objects.ts @@ -1,6 +1,5 @@ import * as Ably from 'ably'; import Objects from 'ably/objects'; -import { CustomRoot } from './ably.config'; import { createSandboxAblyAPIKey } from './sandbox'; // Fix for "type 'typeof globalThis' has no index signature" error: @@ -9,6 +8,26 @@ declare module globalThis { var testAblyPackage: () => Promise; } +type CustomRoot = { + numberKey: number; + stringKey: string; + booleanKey: boolean; + couldBeUndefined?: string; + mapKey: Ably.LiveMap<{ + foo: 'bar'; + nestedMap?: Ably.LiveMap<{ + baz: 'qux'; + }>; + }>; + counterKey: Ably.LiveCounter; +}; + +declare global { + export interface AblyObjectsTypes { + root: CustomRoot; + } +} + type ExplicitRootType = { someOtherKey: string; }; From 92c7cfc06f0dce88af1e792065faa8f9d90c7541 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 28 Apr 2025 14:54:18 +0100 Subject: [PATCH 155/166] Clarify that user provided typings for Objects don't provide runtime checks --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b566b683bf..f62b9a8f12 100644 --- a/README.md +++ b/README.md @@ -886,7 +886,7 @@ declare global { } ``` -This will enable code completion and editor hints when interacting with the Objects API: +Note that using TypeScript typings for Objects does not provide runtime type checking; instead, it enables code completion and editor hints (if supported by your IDE) when interacting with the Objects API: ```typescript const root = await objects.getRoot(); // uses types defined by global AblyObjectsTypes interface by default From 51063291074a3fa305a778b00781f16dc45958fc Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 28 Apr 2025 17:34:16 +0100 Subject: [PATCH 156/166] Revert "Fix incorrect `moduleResolution` in `tsconfig.json` for package tests" This reverts commit d59e16710f51421ccf2dba7f53956eccb1d1283c. The `moduleResolution` option should be `bundler`, just like it was set in the original fecf2dfacaa6e849686a32145c8a715163d34d4a commit. The supposed TypeScript error "Option '--resolveJsonModule' cannot be specified without 'node' module resolution strategy." is actually coming from the root folder TypeScript installation, not the one used for the `package` test folder. The `--moduleResolution bundler` option was added in TypeScript 5.0 [1] and in the root we have TypeScript ^4.9 installed, compared to TypeScript ^5.2 used for `package`` folder. See PR review comment for more context [2]. [1] https://www.typescriptlang.org/docs/handbook/release-notes/typescript-5-0.html#--moduleresolution-bundler [2] https://github.com/ably/ably-js/pull/2007#discussion_r2054359256 --- test/package/browser/template/src/tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/package/browser/template/src/tsconfig.json b/test/package/browser/template/src/tsconfig.json index 925607050a..1640a1da14 100644 --- a/test/package/browser/template/src/tsconfig.json +++ b/test/package/browser/template/src/tsconfig.json @@ -5,7 +5,7 @@ "resolveJsonModule": true, "esModuleInterop": true, "module": "esnext", - "moduleResolution": "node", + "moduleResolution": "bundler", "jsx": "react-jsx" } } From 44aae8e9d7d5fe200c9fb048f34a03b92b782036 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 6 May 2025 10:48:28 +0100 Subject: [PATCH 157/166] Remove StateValue from ObjectMessage Leaf values for objects are now stored in type keys on ObjectData object instead of a single `value` key with a union type. The new approach caused an issue with comet transports (only implemented in ably-js) as comet transports do not support msgpack protocol. We can't rely on the usual `client.options.useBinaryProtocol` option to identify the protocol used, as comet transport changes the selected protocol in its implementation. Instead we get the current active transport object and its format. Resolves PUB-1666 --- ably.d.ts | 8 +- src/common/lib/client/realtimechannel.ts | 15 +- src/common/lib/transport/connectionmanager.ts | 4 + src/plugins/objects/livemap.ts | 112 ++++-- src/plugins/objects/objectmessage.ts | 186 ++++++---- test/common/modules/private_api_recorder.js | 1 + test/realtime/objects.test.js | 336 +++++++++++------- 7 files changed, 435 insertions(+), 227 deletions(-) diff --git a/ably.d.ts b/ably.d.ts index f2f5301b32..df8abdcef4 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -2374,9 +2374,9 @@ declare global { /** * Represents the type of data stored in a {@link LiveMap}. - * It maps string keys to scalar values ({@link ObjectValue}), or other {@link LiveObject | LiveObjects}. + * It maps string keys to primitive values ({@link PrimitiveObjectValue}), or other {@link LiveObject | LiveObjects}. */ -export type LiveMapType = { [key: string]: ObjectValue | LiveMap | LiveCounter | undefined }; +export type LiveMapType = { [key: string]: PrimitiveObjectValue | LiveMap | LiveCounter | undefined }; /** * The default type for the `root` object for Objects on a channel, based on the globally defined {@link AblyObjectsTypes} interface. @@ -2502,7 +2502,7 @@ export declare interface BatchContextLiveCounter { * Conflicts in a LiveMap are automatically resolved with last-write-wins (LWW) semantics, * meaning that if two clients update the same key in the map, the update with the most recent timestamp wins. * - * Keys must be strings. Values can be another {@link LiveObject}, or a primitive type, such as a string, number, boolean, or binary data (see {@link ObjectValue}). + * Keys must be strings. Values can be another {@link LiveObject}, or a primitive type, such as a string, number, boolean, or binary data (see {@link PrimitiveObjectValue}). */ export declare interface LiveMap extends LiveObject> { /** @@ -2589,7 +2589,7 @@ export declare interface LiveMapUpdate extends LiveObject * * For binary data, the resulting type depends on the platform (`Buffer` in Node.js, `ArrayBuffer` elsewhere). */ -export type ObjectValue = string | number | boolean | Buffer | ArrayBuffer; +export type PrimitiveObjectValue = string | number | boolean | Buffer | ArrayBuffer; /** * The `LiveCounter` class represents a counter that can be incremented or decremented and is synchronized across clients in realtime. diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index 25dc1d19cb..8a95b2ce52 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -3,12 +3,7 @@ import ProtocolMessage, { fromValues as protocolMessageFromValues } from '../typ import EventEmitter from '../util/eventemitter'; import * as Utils from '../util/utils'; import Logger from '../util/logger'; -import { - EncodingDecodingContext, - CipherOptions, - populateFieldsFromParent, - MessageEncoding, -} from '../types/basemessage'; +import { EncodingDecodingContext, CipherOptions, populateFieldsFromParent } from '../types/basemessage'; import Message, { getMessagesSize, encodeArray as encodeMessagesArray } from '../types/message'; import ChannelStateChange from './channelstatechange'; import ErrorInfo, { PartialErrorInfo } from '../types/errorinfo'; @@ -626,11 +621,15 @@ class RealtimeChannel extends EventEmitter { } populateFieldsFromParent(message); - const options = this.channelOptions; const objectMessages = message.state; + // need to use the active protocol format instead of just client's useBinaryProtocol option, + // as comet transport does not support msgpack and will default to json without changing useBinaryProtocol. + // message processing is done in the same event loop tick up until this point, + // so we can reliably expect an active protocol to exist and be the one that received the object message. + const format = this.client.connection.connectionManager.getActiveTransportFormat()!; await Promise.all( objectMessages.map((om) => - this.client._objectsPlugin!.ObjectMessage.decode(om, options, MessageEncoding, this.logger, Logger, Utils), + this.client._objectsPlugin!.ObjectMessage.decode(om, this.client, this.logger, Logger, Utils, format), ), ); diff --git a/src/common/lib/transport/connectionmanager.ts b/src/common/lib/transport/connectionmanager.ts index 861647e86e..3017dd7c71 100644 --- a/src/common/lib/transport/connectionmanager.ts +++ b/src/common/lib/transport/connectionmanager.ts @@ -949,6 +949,10 @@ class ConnectionManager extends EventEmitter { this.clearSessionRecoverData(); } + getActiveTransportFormat(): Utils.Format | undefined { + return this.activeProtocol?.getTransport().format; + } + /********************* * state management *********************/ diff --git a/src/plugins/objects/livemap.ts b/src/plugins/objects/livemap.ts index 22eaa0a29e..1c9a3b58b0 100644 --- a/src/plugins/objects/livemap.ts +++ b/src/plugins/objects/livemap.ts @@ -1,5 +1,6 @@ import { dequal } from 'dequal'; +import type { Bufferlike } from 'common/platform'; import type * as API from '../../../ably'; import { DEFAULTS } from './defaults'; import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; @@ -12,23 +13,27 @@ import { ObjectOperation, ObjectOperationAction, ObjectState, - ObjectValue, } from './objectmessage'; import { Objects } from './objects'; +export type PrimitiveObjectValue = string | number | boolean | Bufferlike; + export interface ObjectIdObjectData { /** A reference to another object, used to support composable object structures. */ objectId: string; } export interface ValueObjectData { - /** - * The encoding the client should use to interpret the value. - * Analogous to the `encoding` field on the `Message` and `PresenceMessage` types. - */ + /** Can be set by the client to indicate that value in `string` or `bytes` field have an encoding. */ encoding?: string; - /** A concrete leaf value in the object graph. */ - value: ObjectValue; + /** A primitive boolean leaf value in the object graph. Only one value field can be set. */ + boolean?: boolean; + /** A primitive binary leaf value in the object graph. Only one value field can be set. */ + bytes?: Bufferlike; + /** A primitive number leaf value in the object graph. Only one value field can be set. */ + number?: number; + /** A primitive string leaf value in the object graph. Only one value field can be set. */ + string?: string; } export type ObjectData = ObjectIdObjectData | ValueObjectData; @@ -109,10 +114,23 @@ export class LiveMap extends LiveObject extends LiveObject = {}; Object.entries(entries ?? {}).forEach(([key, value]) => { - const objectData: ObjectData = - value instanceof LiveObject - ? ({ objectId: value.getObjectId() } as ObjectIdObjectData) - : ({ value } as ValueObjectData); + let objectData: ObjectData; + if (value instanceof LiveObject) { + const typedObjectData: ObjectIdObjectData = { objectId: value.getObjectId() }; + objectData = typedObjectData; + } else { + const typedObjectData: ValueObjectData = {}; + if (typeof value === 'string') { + typedObjectData.string = value; + } else if (typeof value === 'number') { + typedObjectData.number = value; + } else if (typeof value === 'boolean') { + typedObjectData.boolean = value; + } else { + typedObjectData.bytes = value as Bufferlike; + } + objectData = typedObjectData; + } mapEntries[key] = { data: objectData, @@ -666,7 +697,14 @@ export class LiveMap extends LiveObject extends LiveObject extends LiveObject { let liveData: ObjectData; - if (typeof entry.data.objectId !== 'undefined') { + + if (!this._client.Utils.isNil(entry.data.objectId)) { liveData = { objectId: entry.data.objectId } as ObjectIdObjectData; } else { - liveData = { encoding: entry.data.encoding, value: entry.data.value } as ValueObjectData; + liveData = { + encoding: entry.data.encoding, + boolean: entry.data.boolean, + bytes: entry.data.bytes, + number: entry.data.number, + string: entry.data.string, + } as ValueObjectData; } const liveDataEntry: LiveMapEntry = { @@ -802,14 +853,25 @@ export class LiveMap extends LiveObject { data: any; encoding?: string | null }; +export type EncodeInitialValueFunction = ( + data: any, + encoding?: string | null, +) => { data: any; encoding?: string | null }; + +export type EncodeObjectDataFunction = (data: ObjectData) => ObjectData; export enum ObjectOperationAction { MAP_CREATE = 0, @@ -20,20 +24,21 @@ export enum MapSemantics { LWW = 0, } -/** An ObjectValue represents a concrete leaf value in the object graph. */ -export type ObjectValue = string | number | boolean | Bufferlike; - /** An ObjectData represents a value in an object on a channel. */ export interface ObjectData { /** A reference to another object, used to support composable object structures. */ objectId?: string; - /** - * The encoding the client should use to interpret the value. - * Analogous to the `encoding` field on the `Message` and `PresenceMessage` types. - */ + + /** Can be set by the client to indicate that value in `string` or `bytes` field have an encoding. */ encoding?: string; - /** A concrete leaf value in the object graph. */ - value?: ObjectValue; + /** A primitive boolean leaf value in the object graph. Only one value field can be set. */ + boolean?: boolean; + /** A primitive binary leaf value in the object graph. Only one value field can be set. */ + bytes?: Bufferlike; + /** A primitive number leaf value in the object graph. Only one value field can be set. */ + number?: number; + /** A primitive string leaf value in the object graph. Only one value field can be set. */ + string?: string; } /** A MapOp describes an operation to be applied to a Map object. */ @@ -141,6 +146,9 @@ export interface ObjectState { counter?: ObjectCounter; } +// TODO: tidy up encoding/decoding logic for ObjectMessage: +// Should have separate WireObjectMessage with the correct types received from the server, do the necessary encoding/decoding there. +// For reference, see WireMessage and WirePresenceMessage /** * @internal */ @@ -180,7 +188,7 @@ export class ObjectMessage { * Uses encoding functions from regular `Message` processing. */ static async encode(message: ObjectMessage, messageEncoding: typeof MessageEncoding): Promise { - const encodeFn: EncodeFunction = (data, encoding) => { + const encodeInitialValueFn: EncodeInitialValueFunction = (data, encoding) => { const { data: encodedData, encoding: newEncoding } = messageEncoding.encodeData(data, encoding); return { @@ -189,46 +197,60 @@ export class ObjectMessage { }; }; + const encodeObjectDataFn: EncodeObjectDataFunction = (data) => { + // TODO: support encoding JSON objects as a JSON string on "string" property with an encoding of "json" + // https://ably.atlassian.net/browse/PUB-1667 + // for now just return values as they are + + return data; + }; + message.operation = message.operation - ? ObjectMessage._encodeObjectOperation(message.operation, encodeFn) + ? ObjectMessage._encodeObjectOperation(message.operation, encodeObjectDataFn, encodeInitialValueFn) + : undefined; + message.object = message.object + ? ObjectMessage._encodeObjectState(message.object, encodeObjectDataFn, encodeInitialValueFn) : undefined; - message.object = message.object ? ObjectMessage._encodeObjectState(message.object, encodeFn) : undefined; return message; } /** - * Mutates the provided ObjectMessage and decodes all data entries in the message + * Mutates the provided ObjectMessage and decodes all data entries in the message. + * + * Format is used to decode the bytes value as it's implicitly encoded depending on the protocol used: + * - json: bytes are base64 encoded string + * - msgpack: bytes have a binary representation and don't need to be decoded */ static async decode( message: ObjectMessage, - inputContext: ChannelOptions, - messageEncoding: typeof MessageEncoding, + client: BaseClient, logger: Logger, LoggerClass: typeof Logger, utils: typeof Utils, + format: Utils.Format | undefined, ): Promise { // TODO: decide how to handle individual errors from decoding values. currently we throw first ever error we get try { if (message.object?.map?.entries) { - await ObjectMessage._decodeMapEntries(message.object.map.entries, inputContext, messageEncoding); + await ObjectMessage._decodeMapEntries(message.object.map.entries, client, format); } if (message.object?.createOp?.map?.entries) { - await ObjectMessage._decodeMapEntries(message.object.createOp.map.entries, inputContext, messageEncoding); + await ObjectMessage._decodeMapEntries(message.object.createOp.map.entries, client, format); } - if (message.object?.createOp?.mapOp?.data && 'value' in message.object.createOp.mapOp.data) { - await ObjectMessage._decodeObjectData(message.object.createOp.mapOp.data, inputContext, messageEncoding); + if (message.object?.createOp?.mapOp?.data) { + await ObjectMessage._decodeObjectData(message.object.createOp.mapOp.data, client, format); } if (message.operation?.map?.entries) { - await ObjectMessage._decodeMapEntries(message.operation.map.entries, inputContext, messageEncoding); + await ObjectMessage._decodeMapEntries(message.operation.map.entries, client, format); } - if (message.operation?.mapOp?.data && 'value' in message.operation.mapOp.data) { - await ObjectMessage._decodeObjectData(message.operation.mapOp.data, inputContext, messageEncoding); + if (message.operation?.mapOp?.data) { + await ObjectMessage._decodeObjectData(message.operation.mapOp.data, client, format); } } catch (error) { LoggerClass.logAction(logger, LoggerClass.LOG_ERROR, 'ObjectMessage.decode()', utils.inspectError(error)); @@ -296,59 +318,69 @@ export class ObjectMessage { private static async _decodeMapEntries( mapEntries: Record, - inputContext: ChannelOptions, - messageEncoding: typeof MessageEncoding, + client: BaseClient, + format: Utils.Format | undefined, ): Promise { for (const entry of Object.values(mapEntries)) { - await ObjectMessage._decodeObjectData(entry.data, inputContext, messageEncoding); + await ObjectMessage._decodeObjectData(entry.data, client, format); } } private static async _decodeObjectData( objectData: ObjectData, - inputContext: ChannelOptions, - messageEncoding: typeof MessageEncoding, + client: BaseClient, + format: Utils.Format | undefined, ): Promise { - const { data, encoding, error } = await messageEncoding.decodeData( - objectData.value, - objectData.encoding, - inputContext, - ); - objectData.value = data; - objectData.encoding = encoding ?? undefined; - - if (error) { - throw error; + // TODO: support decoding JSON objects stored as a JSON string with an encoding of "json" + // https://ably.atlassian.net/browse/PUB-1667 + // currently we check only the "bytes" field: + // - if connection is msgpack - "bytes" was received as msgpack encoded bytes, no need to decode, it's already a buffer + // - if connection is json - "bytes" was received as a base64 string, need to decode it to a buffer + + if (format !== 'msgpack' && objectData.bytes != null) { + // connection is using JSON protocol, decode bytes value + objectData.bytes = client.Platform.BufferUtils.base64Decode(String(objectData.bytes)); } } - private static _encodeObjectOperation(objectOperation: ObjectOperation, encodeFn: EncodeFunction): ObjectOperation { + private static _encodeObjectOperation( + objectOperation: ObjectOperation, + encodeObjectDataFn: EncodeObjectDataFunction, + encodeInitialValueFn: EncodeInitialValueFunction, + ): ObjectOperation { // deep copy "objectOperation" object so we can modify the copy here. // buffer values won't be correctly copied, so we will need to set them again explicitly. const objectOperationCopy = JSON.parse(JSON.stringify(objectOperation)) as ObjectOperation; - if (objectOperationCopy.mapOp?.data && 'value' in objectOperationCopy.mapOp.data) { + if (objectOperationCopy.mapOp?.data) { // use original "objectOperation" object when encoding values, so we have access to the original buffer values. - objectOperationCopy.mapOp.data = ObjectMessage._encodeObjectData(objectOperation.mapOp?.data!, encodeFn); + objectOperationCopy.mapOp.data = ObjectMessage._encodeObjectData( + objectOperation.mapOp?.data!, + encodeObjectDataFn, + ); } if (objectOperationCopy.map?.entries) { Object.entries(objectOperationCopy.map.entries).forEach(([key, entry]) => { // use original "objectOperation" object when encoding values, so we have access to original buffer values. - entry.data = ObjectMessage._encodeObjectData(objectOperation?.map?.entries?.[key].data!, encodeFn); + entry.data = ObjectMessage._encodeObjectData(objectOperation?.map?.entries?.[key].data!, encodeObjectDataFn); }); } if (objectOperation.initialValue) { // use original "objectOperation" object so we have access to the original buffer value - const { data: encodedInitialValue } = encodeFn(objectOperation.initialValue); + const { data: encodedInitialValue } = encodeInitialValueFn(objectOperation.initialValue); objectOperationCopy.initialValue = encodedInitialValue; } return objectOperationCopy; } - private static _encodeObjectState(objectState: ObjectState, encodeFn: EncodeFunction): ObjectState { + private static _encodeObjectState( + objectState: ObjectState, + encodeObjectDataFn: EncodeObjectDataFunction, + encodeInitialValueFn: EncodeInitialValueFunction, + ): ObjectState { // deep copy "objectState" object so we can modify the copy here. // buffer values won't be correctly copied, so we will need to set them again explicitly. const objectStateCopy = JSON.parse(JSON.stringify(objectState)) as ObjectState; @@ -356,26 +388,25 @@ export class ObjectMessage { if (objectStateCopy.map?.entries) { Object.entries(objectStateCopy.map.entries).forEach(([key, entry]) => { // use original "objectState" object when encoding values, so we have access to original buffer values. - entry.data = ObjectMessage._encodeObjectData(objectState?.map?.entries?.[key].data!, encodeFn); + entry.data = ObjectMessage._encodeObjectData(objectState?.map?.entries?.[key].data!, encodeObjectDataFn); }); } if (objectStateCopy.createOp) { // use original "objectState" object when encoding values, so we have access to original buffer values. - objectStateCopy.createOp = ObjectMessage._encodeObjectOperation(objectState.createOp!, encodeFn); + objectStateCopy.createOp = ObjectMessage._encodeObjectOperation( + objectState.createOp!, + encodeObjectDataFn, + encodeInitialValueFn, + ); } return objectStateCopy; } - private static _encodeObjectData(data: ObjectData, encodeFn: EncodeFunction): ObjectData { - const { data: encodedValue, encoding: newEncoding } = encodeFn(data?.value, data?.encoding); - - return { - ...data, - value: encodedValue, - encoding: newEncoding ?? undefined, - }; + private static _encodeObjectData(data: ObjectData, encodeFn: EncodeObjectDataFunction): ObjectData { + const encodedData = encodeFn(data); + return encodedData; } /** @@ -391,7 +422,7 @@ export class ObjectMessage { operation?: ObjectOperation; objectState?: ObjectState; } { - const encodeFn: EncodeFunction = (data, encoding) => { + const encodeInitialValueFn: EncodeInitialValueFunction = (data, encoding) => { const { data: encodedData, encoding: newEncoding } = messageEncoding.encodeDataForWire(data, encoding, format); return { data: encodedData, @@ -399,10 +430,32 @@ export class ObjectMessage { }; }; + const encodeObjectDataFn: EncodeObjectDataFunction = (data) => { + // TODO: support encoding JSON objects as a JSON string on "string" property with an encoding of "json" + // https://ably.atlassian.net/browse/PUB-1667 + // currently we check only the "bytes" field: + // - if connection is msgpack - "bytes" will will be sent as msgpack bytes, no need to encode here + // - if connection is json - "bytes" will be encoded as a base64 string + + let encodedBytes: any = data.bytes; + if (data.bytes != null) { + const result = messageEncoding.encodeDataForWire(data.bytes, data.encoding, format); + encodedBytes = result.data; + // no need to change the encoding + } + + return { + ...data, + bytes: encodedBytes, + }; + }; + const encodedOperation = message.operation - ? ObjectMessage._encodeObjectOperation(message.operation, encodeFn) + ? ObjectMessage._encodeObjectOperation(message.operation, encodeObjectDataFn, encodeInitialValueFn) + : undefined; + const encodedObjectState = message.object + ? ObjectMessage._encodeObjectState(message.object, encodeObjectDataFn, encodeInitialValueFn) : undefined; - const encodedObjectState = message.object ? ObjectMessage._encodeObjectState(message.object, encodeFn) : undefined; return { operation: encodedOperation, @@ -566,14 +619,19 @@ export class ObjectMessage { private _getObjectDataSize(data: ObjectData): number { let size = 0; - if (data.value) { - size += this._getObjectValueSize(data.value); + if (data.boolean != null) { + size += this._utils.dataSizeBytes(data.boolean); + } + if (data.bytes != null) { + size += this._utils.dataSizeBytes(data.bytes); + } + if (data.number != null) { + size += this._utils.dataSizeBytes(data.number); + } + if (data.string != null) { + size += this._utils.dataSizeBytes(data.string); } return size; } - - private _getObjectValueSize(value: ObjectValue): number { - return this._utils.dataSizeBytes(value); - } } diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index 8aec585500..86e6d099f8 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -112,6 +112,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'read.realtime.options.maxMessageSize', 'read.realtime.options.realtimeHost', 'read.realtime.options.token', + 'read.realtime.options.useBinaryProtocol', 'read.rest._currentFallback', 'read.rest._currentFallback.host', 'read.rest._currentFallback.validUntil', diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js index 05ef921c12..e3b4dc1ce2 100644 --- a/test/realtime/objects.test.js +++ b/test/realtime/objects.test.js @@ -198,7 +198,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function channel: testChannel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'stringKey', data: { value: 'stringValue' } })], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'stringKey', data: { string: 'stringValue' } })], }); const publishChannel = publishClient.channels.get('channel'); @@ -403,7 +403,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function objectsHelper.mapObject({ objectId: 'root', siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, - initialEntries: { key: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 1 } } }, + initialEntries: { key: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { number: 1 } } }, }), ], }); @@ -623,28 +623,16 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ); const primitiveKeyData = [ - { key: 'stringKey', data: { value: 'stringValue' }, restData: { string: 'stringValue' } }, - { key: 'emptyStringKey', data: { value: '' }, restData: { string: '' } }, - { - key: 'bytesKey', - data: { value: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9', encoding: 'base64' }, - restData: { bytes: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9' }, - }, - { key: 'emptyBytesKey', data: { value: '', encoding: 'base64' }, restData: { bytes: '' } }, - { - key: 'maxSafeIntegerKey', - data: { value: Number.MAX_SAFE_INTEGER }, - restData: { number: Number.MAX_SAFE_INTEGER }, - }, - { - key: 'negativeMaxSafeIntegerKey', - data: { value: -Number.MAX_SAFE_INTEGER }, - restData: { number: -Number.MAX_SAFE_INTEGER }, - }, - { key: 'numberKey', data: { value: 1 }, restData: { number: 1 } }, - { key: 'zeroKey', data: { value: 0 }, restData: { number: 0 } }, - { key: 'trueKey', data: { value: true }, restData: { boolean: true } }, - { key: 'falseKey', data: { value: false }, restData: { boolean: false } }, + { key: 'stringKey', data: { string: 'stringValue' } }, + { key: 'emptyStringKey', data: { string: '' } }, + { key: 'bytesKey', data: { bytes: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9' } }, + { key: 'emptyBytesKey', data: { bytes: '' } }, + { key: 'maxSafeIntegerKey', data: { number: Number.MAX_SAFE_INTEGER } }, + { key: 'negativeMaxSafeIntegerKey', data: { number: -Number.MAX_SAFE_INTEGER } }, + { key: 'numberKey', data: { number: 1 } }, + { key: 'zeroKey', data: { number: 0 } }, + { key: 'trueKey', data: { boolean: true } }, + { key: 'falseKey', data: { boolean: false } }, ]; const primitiveMapsFixtures = [ { name: 'emptyMap' }, @@ -655,7 +643,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function return acc; }, {}), restData: primitiveKeyData.reduce((acc, v) => { - acc[v.key] = v.restData; + acc[v.key] = v.data; return acc; }, {}), }, @@ -704,7 +692,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function initialEntries: { map: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: mapId } }, counter: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId } }, - foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'bar' } }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'bar' } }, }, }), ], @@ -760,7 +748,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, initialEntries: { counter: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId } }, - foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'bar' } }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'bar' } }, }, }), ], @@ -879,16 +867,17 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ); Object.entries(fixture.entries ?? {}).forEach(([key, keyData]) => { - if (keyData.data.encoding) { + if (keyData.data.bytes != null) { helper.recordPrivateApi('call.BufferUtils.base64Decode'); helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( - BufferUtils.areBuffersEqual(mapObj.get(key), BufferUtils.base64Decode(keyData.data.value)), + BufferUtils.areBuffersEqual(mapObj.get(key), BufferUtils.base64Decode(keyData.data.bytes)), `Check map "${mapKey}" has correct value for "${key}" key`, ).to.be.true; } else { + const valueType = typeof mapObj.get(key); expect(mapObj.get(key)).to.equal( - keyData.data.value, + keyData.data[valueType], `Check map "${mapKey}" has correct value for "${key}" key`, ); } @@ -994,7 +983,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function channel, serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb', - state: [objectsHelper.mapSetOp({ objectId: mapId, key: 'foo', data: { value: 'bar' } })], + state: [objectsHelper.mapSetOp({ objectId: mapId, key: 'foo', data: { string: 'bar' } })], }); await objectsHelper.processObjectOperationMessageOnChannel({ channel, @@ -1021,7 +1010,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function objectsHelper.mapCreateOp({ objectId: mapIds[i], entries: { - baz: { timeserial: serial, data: { value: 'qux' } }, + baz: { timeserial: serial, data: { string: 'qux' } }, }, }), ], @@ -1078,7 +1067,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function objectsHelper.mapSetRestOp({ objectId: 'root', key: keyData.key, - value: keyData.restData, + value: keyData.data, }), ), ), @@ -1087,16 +1076,17 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // check everything is applied correctly primitiveKeyData.forEach((keyData) => { - if (keyData.data.encoding) { + if (keyData.data.bytes != null) { helper.recordPrivateApi('call.BufferUtils.base64Decode'); helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( - BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.bytes)), `Check root has correct value for "${keyData.key}" key after MAP_SET op`, ).to.be.true; } else { + const valueType = typeof root.get(keyData.key); expect(root.get(keyData.key)).to.equal( - keyData.data.value, + keyData.data[valueType], `Check root has correct value for "${keyData.key}" key after MAP_SET op`, ); } @@ -1178,12 +1168,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function objectsHelper.mapCreateOp({ objectId: mapId, entries: { - foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo3: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo5: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo6: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo3: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo5: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo6: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, }, }), ], @@ -1208,7 +1198,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function channel, serial, siteCode, - state: [objectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { value: 'baz' } })], + state: [objectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { string: 'baz' } })], }); } @@ -1310,12 +1300,12 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function objectsHelper.mapCreateOp({ objectId: mapId, entries: { - foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo3: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo5: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo6: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, + foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo3: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo5: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo6: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, }, }), ], @@ -1914,7 +1904,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function channel, serial: lexicoTimeserial('aaa', 3, 0), siteCode: 'aaa', - state: [objectsHelper.mapSetOp({ objectId: mapId1, key: 'baz', data: { value: 'qux' } })], + state: [objectsHelper.mapSetOp({ objectId: mapId1, key: 'baz', data: { string: 'qux' } })], }); await objectsHelper.processObjectOperationMessageOnChannel({ channel, @@ -1946,7 +1936,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { description: 'object operation messages are buffered during OBJECT_SYNC sequence', action: async (ctx) => { - const { root, objectsHelper, channel } = ctx; + const { root, objectsHelper, channel, client, helper } = ctx; // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages await objectsHelper.processObjectStateMessageOnChannel({ @@ -1956,15 +1946,23 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // inject operations, it should not be applied as sync is in progress await Promise.all( - primitiveKeyData.map((keyData) => - objectsHelper.processObjectOperationMessageOnChannel({ + primitiveKeyData.map(async (keyData) => { + // copy data object as library will modify it + const data = { ...keyData.data }; + helper.recordPrivateApi('read.realtime.options.useBinaryProtocol'); + if (data.bytes != null && client.options.useBinaryProtocol) { + // decode base64 data to binary for binary protocol + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + data.bytes = BufferUtils.base64Decode(data.bytes); + } + + return objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa', - // copy data object as library will modify it - state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } })], - }), - ), + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data })], + }); + }), ); // check root doesn't have data from operations @@ -1978,7 +1976,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { description: 'buffered object operation messages are applied when OBJECT_SYNC sequence ends', action: async (ctx) => { - const { root, objectsHelper, channel, helper } = ctx; + const { root, objectsHelper, channel, helper, client } = ctx; // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages await objectsHelper.processObjectStateMessageOnChannel({ @@ -1988,15 +1986,23 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // inject operations, they should be applied when sync ends await Promise.all( - primitiveKeyData.map((keyData, i) => - objectsHelper.processObjectOperationMessageOnChannel({ + primitiveKeyData.map(async (keyData, i) => { + // copy data object as library will modify it + const data = { ...keyData.data }; + helper.recordPrivateApi('read.realtime.options.useBinaryProtocol'); + if (data.bytes != null && client.options.useBinaryProtocol) { + // decode base64 data to binary for binary protocol + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + data.bytes = BufferUtils.base64Decode(data.bytes); + } + + return objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', - // copy data object as library will modify it - state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } })], - }), - ), + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data })], + }); + }), ); // end the sync with empty cursor @@ -2007,16 +2013,17 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // check everything is applied correctly primitiveKeyData.forEach((keyData) => { - if (keyData.data.encoding) { + if (keyData.data.bytes != null) { helper.recordPrivateApi('call.BufferUtils.base64Decode'); helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( - BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.bytes)), `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, ).to.be.true; } else { + const valueType = typeof root.get(keyData.key); expect(root.get(keyData.key)).to.equal( - keyData.data.value, + keyData.data[valueType], `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, ); } @@ -2027,7 +2034,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { description: 'buffered object operation messages are discarded when new OBJECT_SYNC sequence starts', action: async (ctx) => { - const { root, objectsHelper, channel } = ctx; + const { root, objectsHelper, channel, client, helper } = ctx; // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages await objectsHelper.processObjectStateMessageOnChannel({ @@ -2037,15 +2044,23 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // inject operations, expect them to be discarded when sync with new sequence id starts await Promise.all( - primitiveKeyData.map((keyData, i) => - objectsHelper.processObjectOperationMessageOnChannel({ + primitiveKeyData.map(async (keyData, i) => { + // copy data object as library will modify it + const data = { ...keyData.data }; + helper.recordPrivateApi('read.realtime.options.useBinaryProtocol'); + if (data.bytes != null && client.options.useBinaryProtocol) { + // decode base64 data to binary for binary protocol + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + data.bytes = BufferUtils.base64Decode(data.bytes); + } + + return objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', - // copy data object as library will modify it - state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } })], - }), - ), + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data })], + }); + }), ); // start new sync with new sequence id @@ -2059,7 +2074,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function channel, serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb', - state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { value: 'bar' } })], + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { string: 'bar' } })], }); // end sync @@ -2106,14 +2121,14 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ccc: lexicoTimeserial('ccc', 5, 0), }, materialisedEntries: { - foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo3: { timeserial: lexicoTimeserial('ccc', 5, 0), data: { value: 'bar' } }, - foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { value: 'bar' } }, - foo5: { timeserial: lexicoTimeserial('bbb', 2, 0), data: { value: 'bar' } }, - foo6: { timeserial: lexicoTimeserial('ccc', 2, 0), data: { value: 'bar' } }, - foo7: { timeserial: lexicoTimeserial('ccc', 0, 0), data: { value: 'bar' } }, - foo8: { timeserial: lexicoTimeserial('ccc', 0, 0), data: { value: 'bar' } }, + foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo3: { timeserial: lexicoTimeserial('ccc', 5, 0), data: { string: 'bar' } }, + foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo5: { timeserial: lexicoTimeserial('bbb', 2, 0), data: { string: 'bar' } }, + foo6: { timeserial: lexicoTimeserial('ccc', 2, 0), data: { string: 'bar' } }, + foo7: { timeserial: lexicoTimeserial('ccc', 0, 0), data: { string: 'bar' } }, + foo8: { timeserial: lexicoTimeserial('ccc', 0, 0), data: { string: 'bar' } }, }, }), objectsHelper.counterObject({ @@ -2153,7 +2168,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function channel, serial, siteCode, - state: [objectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { value: 'baz' } })], + state: [objectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { string: 'baz' } })], }); } @@ -2210,7 +2225,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function description: 'subsequent object operation messages are applied immediately after OBJECT_SYNC ended and buffers are applied', action: async (ctx) => { - const { root, objectsHelper, channel, channelName, helper } = ctx; + const { root, objectsHelper, channel, channelName, helper, client } = ctx; // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages await objectsHelper.processObjectStateMessageOnChannel({ @@ -2220,15 +2235,23 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // inject operations, they should be applied when sync ends await Promise.all( - primitiveKeyData.map((keyData, i) => - objectsHelper.processObjectOperationMessageOnChannel({ + primitiveKeyData.map(async (keyData, i) => { + // copy data object as library will modify it + const data = { ...keyData.data }; + helper.recordPrivateApi('read.realtime.options.useBinaryProtocol'); + if (data.bytes != null && client.options.useBinaryProtocol) { + // decode base64 data to binary for binary protocol + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + data.bytes = BufferUtils.base64Decode(data.bytes); + } + + return objectsHelper.processObjectOperationMessageOnChannel({ channel, serial: lexicoTimeserial('aaa', i, 0), siteCode: 'aaa', - // copy data object as library will modify it - state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data: { ...keyData.data } })], - }), - ), + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data })], + }); + }), ); // end the sync with empty cursor @@ -2251,16 +2274,17 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // check buffered operations are applied, as well as the most recent operation outside of the sync sequence is applied primitiveKeyData.forEach((keyData) => { - if (keyData.data.encoding) { + if (keyData.data.bytes != null) { helper.recordPrivateApi('call.BufferUtils.base64Decode'); helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( - BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.bytes)), `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, ).to.be.true; } else { + const valueType = typeof root.get(keyData.key); expect(root.get(keyData.key)).to.equal( - keyData.data.value, + keyData.data[valueType], `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, ); } @@ -2503,8 +2527,18 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const keysUpdatedPromise = Promise.all(primitiveKeyData.map((x) => waitForMapKeyUpdate(root, x.key))); await Promise.all( primitiveKeyData.map(async (keyData) => { - helper.recordPrivateApi('call.BufferUtils.base64Decode'); - const value = keyData.data.encoding ? BufferUtils.base64Decode(keyData.data.value) : keyData.data.value; + let value; + if (keyData.data.bytes != null) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + value = BufferUtils.base64Decode(keyData.data.bytes); + } else if (keyData.data.number != null) { + value = keyData.data.number; + } else if (keyData.data.string != null) { + value = keyData.data.string; + } else if (keyData.data.boolean != null) { + value = keyData.data.boolean; + } + await root.set(keyData.key, value); }), ); @@ -2512,16 +2546,17 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function // check everything is applied correctly primitiveKeyData.forEach((keyData) => { - if (keyData.data.encoding) { + if (keyData.data.bytes != null) { helper.recordPrivateApi('call.BufferUtils.base64Decode'); helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( - BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.value)), + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.bytes)), `Check root has correct value for "${keyData.key}" key after LiveMap.set call`, ).to.be.true; } else { + const valueType = typeof root.get(keyData.key); expect(root.get(keyData.key)).to.equal( - keyData.data.value, + keyData.data[valueType], `Check root has correct value for "${keyData.key}" key after LiveMap.set call`, ); } @@ -2842,10 +2877,18 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function primitiveMapsFixtures.map(async (mapFixture) => { const entries = mapFixture.entries ? Object.entries(mapFixture.entries).reduce((acc, [key, keyData]) => { - helper.recordPrivateApi('call.BufferUtils.base64Decode'); - const value = keyData.data.encoding - ? BufferUtils.base64Decode(keyData.data.value) - : keyData.data.value; + let value; + if (keyData.data.bytes != null) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + value = BufferUtils.base64Decode(keyData.data.bytes); + } else if (keyData.data.number != null) { + value = keyData.data.number; + } else if (keyData.data.string != null) { + value = keyData.data.string; + } else if (keyData.data.boolean != null) { + value = keyData.data.boolean; + } + acc[key] = value; return acc; }, {}) @@ -2868,16 +2911,17 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ); Object.entries(fixture.entries ?? {}).forEach(([key, keyData]) => { - if (keyData.data.encoding) { + if (keyData.data.bytes != null) { helper.recordPrivateApi('call.BufferUtils.base64Decode'); helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); expect( - BufferUtils.areBuffersEqual(map.get(key), BufferUtils.base64Decode(keyData.data.value)), + BufferUtils.areBuffersEqual(map.get(key), BufferUtils.base64Decode(keyData.data.bytes)), `Check map #${i + 1} has correct value for "${key}" key`, ).to.be.true; } else { + const valueType = typeof map.get(key); expect(map.get(key)).to.equal( - keyData.data.value, + keyData.data[valueType], `Check map #${i + 1} has correct value for "${key}" key`, ); } @@ -2990,7 +3034,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function state: [ objectsHelper.mapCreateOp({ objectId: mapId, - entries: { baz: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { value: 'qux' } } }, + entries: { baz: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { string: 'qux' } } }, }), ], }); @@ -3031,8 +3075,8 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function objectsHelper.mapCreateOp({ objectId: mapId, entries: { - foo: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { value: 'qux' } }, - baz: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { value: 'qux' } }, + foo: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { string: 'qux' } }, + baz: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { string: 'qux' } }, }, }), ], @@ -3453,8 +3497,8 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function materialisedEntries: { counter1: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId1 } }, counter2: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId2 } }, - foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'bar' } }, - baz: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'qux' }, tombstone: true }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'bar' } }, + baz: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'qux' }, tombstone: true }, }, }), ], @@ -3511,8 +3555,8 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function materialisedEntries: { counter1: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId1 } }, counter2: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId2 } }, - foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'bar' } }, - baz: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { value: 'qux' }, tombstone: true }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'bar' } }, + baz: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'qux' }, tombstone: true }, }, }), ], @@ -4571,7 +4615,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function operation: { action: 0, objectId: 'object-id', - map: { semantics: 0, entries: { 'key-1': { tombstone: false, data: { value: 'a string' } } } }, + map: { semantics: 0, entries: { 'key-1': { tombstone: false, data: { string: 'a string' } } } }, }, }, MessageEncoding, @@ -4587,7 +4631,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function objectId: 'object-id', map: { semantics: 0, - entries: { 'key-1': { tombstone: false, data: { value: BufferUtils.utf8Encode('my-value') } } }, + entries: { 'key-1': { tombstone: false, data: { bytes: BufferUtils.utf8Encode('my-value') } } }, }, }, }, @@ -4602,12 +4646,38 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function operation: { action: 0, objectId: 'object-id', - map: { semantics: 0, entries: { 'key-1': { tombstone: false, data: { value: true } } } }, + map: { + semantics: 0, + entries: { + 'key-1': { tombstone: false, data: { boolean: true } }, + 'key-2': { tombstone: false, data: { boolean: false } }, + }, + }, }, }, MessageEncoding, ), - expected: Utils.dataSizeBytes('key-1') + 1, + expected: Utils.dataSizeBytes('key-1') + Utils.dataSizeBytes('key-2') + 2, + }, + { + description: 'map create op with double payload', + message: objectMessageFromValues( + { + operation: { + action: 0, + objectId: 'object-id', + map: { + semantics: 0, + entries: { + 'key-1': { tombstone: false, data: { number: 123.456 } }, + 'key-2': { tombstone: false, data: { number: 0 } }, + }, + }, + }, + }, + MessageEncoding, + ), + expected: Utils.dataSizeBytes('key-1') + Utils.dataSizeBytes('key-2') + 16, }, { description: 'map remove op', @@ -4630,7 +4700,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function { description: 'map set operation value=string', message: objectMessageFromValues({ - operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { value: 'my-value' } } }, + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { string: 'my-value' } } }, }), expected: Utils.dataSizeBytes('my-key') + Utils.dataSizeBytes('my-value'), }, @@ -4640,22 +4710,36 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function operation: { action: 1, objectId: 'object-id', - mapOp: { key: 'my-key', data: { value: BufferUtils.utf8Encode('my-value') } }, + mapOp: { key: 'my-key', data: { bytes: BufferUtils.utf8Encode('my-value') } }, }, }), expected: Utils.dataSizeBytes('my-key') + Utils.dataSizeBytes(BufferUtils.utf8Encode('my-value')), }, { - description: 'map set operation value=boolean', + description: 'map set operation value=boolean true', + message: objectMessageFromValues({ + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { boolean: true } } }, + }), + expected: Utils.dataSizeBytes('my-key') + 1, + }, + { + description: 'map set operation value=boolean false', message: objectMessageFromValues({ - operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { value: true } } }, + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { boolean: false } } }, }), expected: Utils.dataSizeBytes('my-key') + 1, }, { description: 'map set operation value=double', message: objectMessageFromValues({ - operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { value: 123.456 } } }, + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { number: 123.456 } } }, + }), + expected: Utils.dataSizeBytes('my-key') + 8, + }, + { + description: 'map set operation value=double 0', + message: objectMessageFromValues({ + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { number: 0 } } }, }), expected: Utils.dataSizeBytes('my-key') + 8, }, @@ -4667,14 +4751,14 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function map: { semantics: 0, entries: { - 'key-1': { tombstone: false, data: { value: 'a string' } }, - 'key-2': { tombstone: true, data: { value: 'another string' } }, + 'key-1': { tombstone: false, data: { string: 'a string' } }, + 'key-2': { tombstone: true, data: { string: 'another string' } }, }, }, createOp: { action: 0, objectId: 'object-id', - map: { semantics: 0, entries: { 'key-3': { tombstone: false, data: { value: 'third string' } } } }, + map: { semantics: 0, entries: { 'key-3': { tombstone: false, data: { string: 'third string' } } } }, }, siteTimeserials: { aaa: lexicoTimeserial('aaa', 111, 111, 1) }, // shouldn't be counted tombstone: false, @@ -4742,7 +4826,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); /** @nospec */ - it('can attach to channel with Objects modes', async function () { + it('can attach to channel with object modes', async function () { const helper = this.test.helper; const client = helper.AblyRealtime(); From 9e225aad1f8c80a4b634093459d895256638138b Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Mon, 5 May 2025 13:30:09 +0100 Subject: [PATCH 158/166] Refactor OBJECT_SYNC sequence tests --- test/realtime/objects.test.js | 376 ++++++++++++++++------------------ 1 file changed, 171 insertions(+), 205 deletions(-) diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js index e3b4dc1ce2..5d60a81f43 100644 --- a/test/realtime/objects.test.js +++ b/test/realtime/objects.test.js @@ -417,211 +417,6 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, client); }); - /** @nospec */ - Helper.testOnAllTransportsAndProtocols( - this, - 'builds object tree from OBJECT_SYNC sequence on channel attachment', - function (options, channelName) { - return async function () { - const helper = this.test.helper; - const client = RealtimeWithObjects(helper, options); - - await helper.monitorConnectionThenCloseAndFinishAsync(async () => { - await waitFixtureChannelIsReady(client); - - const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); - const objects = channel.objects; - - await channel.attach(); - const root = await objects.getRoot(); - - const counterKeys = ['emptyCounter', 'initialValueCounter', 'referencedCounter']; - const mapKeys = ['emptyMap', 'referencedMap', 'valuesMap']; - const rootKeysCount = counterKeys.length + mapKeys.length; - - expect(root, 'Check getRoot() is resolved when OBJECT_SYNC sequence ends').to.exist; - expect(root.size()).to.equal(rootKeysCount, 'Check root has correct number of keys'); - - counterKeys.forEach((key) => { - const counter = root.get(key); - expect(counter, `Check counter at key="${key}" in root exists`).to.exist; - expectInstanceOf( - counter, - 'LiveCounter', - `Check counter at key="${key}" in root is of type LiveCounter`, - ); - }); - - mapKeys.forEach((key) => { - const map = root.get(key); - expect(map, `Check map at key="${key}" in root exists`).to.exist; - expectInstanceOf(map, 'LiveMap', `Check map at key="${key}" in root is of type LiveMap`); - }); - - const valuesMap = root.get('valuesMap'); - const valueMapKeys = [ - 'stringKey', - 'emptyStringKey', - 'bytesKey', - 'emptyBytesKey', - 'numberKey', - 'zeroKey', - 'trueKey', - 'falseKey', - 'mapKey', - ]; - expect(valuesMap.size()).to.equal(valueMapKeys.length, 'Check nested map has correct number of keys'); - valueMapKeys.forEach((key) => { - const value = valuesMap.get(key); - expect(value, `Check value at key="${key}" in nested map exists`).to.exist; - }); - }, client); - }; - }, - ); - - /** @nospec */ - Helper.testOnAllTransportsAndProtocols( - this, - 'LiveCounter is initialized with initial value from OBJECT_SYNC sequence', - function (options, channelName) { - return async function () { - const helper = this.test.helper; - const client = RealtimeWithObjects(helper, options); - - await helper.monitorConnectionThenCloseAndFinishAsync(async () => { - await waitFixtureChannelIsReady(client); - - const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); - const objects = channel.objects; - - await channel.attach(); - const root = await objects.getRoot(); - - const counters = [ - { key: 'emptyCounter', value: 0 }, - { key: 'initialValueCounter', value: 10 }, - { key: 'referencedCounter', value: 20 }, - ]; - - counters.forEach((x) => { - const counter = root.get(x.key); - expect(counter.value()).to.equal(x.value, `Check counter at key="${x.key}" in root has correct value`); - }); - }, client); - }; - }, - ); - - /** @nospec */ - Helper.testOnAllTransportsAndProtocols( - this, - 'LiveMap is initialized with initial value from OBJECT_SYNC sequence', - function (options, channelName) { - return async function () { - const helper = this.test.helper; - const client = RealtimeWithObjects(helper, options); - - await helper.monitorConnectionThenCloseAndFinishAsync(async () => { - await waitFixtureChannelIsReady(client); - - const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); - const objects = channel.objects; - - await channel.attach(); - const root = await objects.getRoot(); - - const emptyMap = root.get('emptyMap'); - expect(emptyMap.size()).to.equal(0, 'Check empty map in root has no keys'); - - const referencedMap = root.get('referencedMap'); - expect(referencedMap.size()).to.equal(1, 'Check referenced map in root has correct number of keys'); - - const counterFromReferencedMap = referencedMap.get('counterKey'); - expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); - - const valuesMap = root.get('valuesMap'); - expect(valuesMap.size()).to.equal(9, 'Check values map in root has correct number of keys'); - - expect(valuesMap.get('stringKey')).to.equal( - 'stringValue', - 'Check values map has correct string value key', - ); - expect(valuesMap.get('emptyStringKey')).to.equal( - '', - 'Check values map has correct empty string value key', - ); - helper.recordPrivateApi('call.BufferUtils.base64Decode'); - helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); - expect( - BufferUtils.areBuffersEqual( - valuesMap.get('bytesKey'), - BufferUtils.base64Decode('eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9'), - ), - 'Check values map has correct bytes value key', - ).to.be.true; - helper.recordPrivateApi('call.BufferUtils.base64Decode'); - helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); - expect( - BufferUtils.areBuffersEqual(valuesMap.get('emptyBytesKey'), BufferUtils.base64Decode('')), - 'Check values map has correct empty bytes value key', - ).to.be.true; - expect(valuesMap.get('numberKey')).to.equal(1, 'Check values map has correct number value key'); - expect(valuesMap.get('zeroKey')).to.equal(0, 'Check values map has correct zero number value key'); - expect(valuesMap.get('trueKey')).to.equal(true, `Check values map has correct 'true' value key`); - expect(valuesMap.get('falseKey')).to.equal(false, `Check values map has correct 'false' value key`); - - const mapFromValuesMap = valuesMap.get('mapKey'); - expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); - }, client); - }; - }, - ); - - /** @nospec */ - Helper.testOnAllTransportsAndProtocols( - this, - 'LiveMap can reference the same object in their keys', - function (options, channelName) { - return async function () { - const helper = this.test.helper; - const client = RealtimeWithObjects(helper, options); - - await helper.monitorConnectionThenCloseAndFinishAsync(async () => { - await waitFixtureChannelIsReady(client); - - const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); - const objects = channel.objects; - - await channel.attach(); - const root = await objects.getRoot(); - - const referencedCounter = root.get('referencedCounter'); - const referencedMap = root.get('referencedMap'); - const valuesMap = root.get('valuesMap'); - - const counterFromReferencedMap = referencedMap.get('counterKey'); - expect(counterFromReferencedMap, 'Check nested counter exists at a key in a map').to.exist; - expectInstanceOf(counterFromReferencedMap, 'LiveCounter', 'Check nested counter is of type LiveCounter'); - expect(counterFromReferencedMap).to.equal( - referencedCounter, - 'Check nested counter is the same object instance as counter on the root', - ); - expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); - - const mapFromValuesMap = valuesMap.get('mapKey'); - expect(mapFromValuesMap, 'Check nested map exists at a key in a map').to.exist; - expectInstanceOf(mapFromValuesMap, 'LiveMap', 'Check nested map is of type LiveMap'); - expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); - expect(mapFromValuesMap).to.equal( - referencedMap, - 'Check nested map is the same object instance as map on the root', - ); - }, client); - }; - }, - ); - const primitiveKeyData = [ { key: 'stringKey', data: { string: 'stringValue' } }, { key: 'emptyStringKey', data: { string: '' } }, @@ -658,6 +453,177 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ]; const objectSyncSequenceScenarios = [ + { + allTransportsAndProtocols: true, + description: 'builds object tree from OBJECT_SYNC sequence on channel attachment', + action: async (ctx) => { + const { client } = ctx; + + await waitFixtureChannelIsReady(client); + + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + const counterKeys = ['emptyCounter', 'initialValueCounter', 'referencedCounter']; + const mapKeys = ['emptyMap', 'referencedMap', 'valuesMap']; + const rootKeysCount = counterKeys.length + mapKeys.length; + + expect(root, 'Check getRoot() is resolved when OBJECT_SYNC sequence ends').to.exist; + expect(root.size()).to.equal(rootKeysCount, 'Check root has correct number of keys'); + + counterKeys.forEach((key) => { + const counter = root.get(key); + expect(counter, `Check counter at key="${key}" in root exists`).to.exist; + expectInstanceOf(counter, 'LiveCounter', `Check counter at key="${key}" in root is of type LiveCounter`); + }); + + mapKeys.forEach((key) => { + const map = root.get(key); + expect(map, `Check map at key="${key}" in root exists`).to.exist; + expectInstanceOf(map, 'LiveMap', `Check map at key="${key}" in root is of type LiveMap`); + }); + + const valuesMap = root.get('valuesMap'); + const valueMapKeys = [ + 'stringKey', + 'emptyStringKey', + 'bytesKey', + 'emptyBytesKey', + 'numberKey', + 'zeroKey', + 'trueKey', + 'falseKey', + 'mapKey', + ]; + expect(valuesMap.size()).to.equal(valueMapKeys.length, 'Check nested map has correct number of keys'); + valueMapKeys.forEach((key) => { + const value = valuesMap.get(key); + expect(value, `Check value at key="${key}" in nested map exists`).to.exist; + }); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveCounter is initialized with initial value from OBJECT_SYNC sequence', + action: async (ctx) => { + const { client } = ctx; + + await waitFixtureChannelIsReady(client); + + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + const counters = [ + { key: 'emptyCounter', value: 0 }, + { key: 'initialValueCounter', value: 10 }, + { key: 'referencedCounter', value: 20 }, + ]; + + counters.forEach((x) => { + const counter = root.get(x.key); + expect(counter.value()).to.equal(x.value, `Check counter at key="${x.key}" in root has correct value`); + }); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveMap is initialized with initial value from OBJECT_SYNC sequence', + action: async (ctx) => { + const { helper, client } = ctx; + + await waitFixtureChannelIsReady(client); + + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + const emptyMap = root.get('emptyMap'); + expect(emptyMap.size()).to.equal(0, 'Check empty map in root has no keys'); + + const referencedMap = root.get('referencedMap'); + expect(referencedMap.size()).to.equal(1, 'Check referenced map in root has correct number of keys'); + + const counterFromReferencedMap = referencedMap.get('counterKey'); + expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); + + const valuesMap = root.get('valuesMap'); + expect(valuesMap.size()).to.equal(9, 'Check values map in root has correct number of keys'); + + expect(valuesMap.get('stringKey')).to.equal('stringValue', 'Check values map has correct string value key'); + expect(valuesMap.get('emptyStringKey')).to.equal('', 'Check values map has correct empty string value key'); + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual( + valuesMap.get('bytesKey'), + BufferUtils.base64Decode('eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9'), + ), + 'Check values map has correct bytes value key', + ).to.be.true; + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual(valuesMap.get('emptyBytesKey'), BufferUtils.base64Decode('')), + 'Check values map has correct empty bytes value key', + ).to.be.true; + expect(valuesMap.get('numberKey')).to.equal(1, 'Check values map has correct number value key'); + expect(valuesMap.get('zeroKey')).to.equal(0, 'Check values map has correct zero number value key'); + expect(valuesMap.get('trueKey')).to.equal(true, `Check values map has correct 'true' value key`); + expect(valuesMap.get('falseKey')).to.equal(false, `Check values map has correct 'false' value key`); + + const mapFromValuesMap = valuesMap.get('mapKey'); + expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveMap can reference the same object in their keys', + action: async (ctx) => { + const { client } = ctx; + + await waitFixtureChannelIsReady(client); + + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + const referencedCounter = root.get('referencedCounter'); + const referencedMap = root.get('referencedMap'); + const valuesMap = root.get('valuesMap'); + + const counterFromReferencedMap = referencedMap.get('counterKey'); + expect(counterFromReferencedMap, 'Check nested counter exists at a key in a map').to.exist; + expectInstanceOf(counterFromReferencedMap, 'LiveCounter', 'Check nested counter is of type LiveCounter'); + expect(counterFromReferencedMap).to.equal( + referencedCounter, + 'Check nested counter is the same object instance as counter on the root', + ); + expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); + + const mapFromValuesMap = valuesMap.get('mapKey'); + expect(mapFromValuesMap, 'Check nested map exists at a key in a map').to.exist; + expectInstanceOf(mapFromValuesMap, 'LiveMap', 'Check nested map is of type LiveMap'); + expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); + expect(mapFromValuesMap).to.equal( + referencedMap, + 'Check nested map is the same object instance as map on the root', + ); + }, + }, + { description: 'OBJECT_SYNC sequence with object state "tombstone" property creates tombstoned object', action: async (ctx) => { From 567d58cce37dcac3fc4396e7189a3e52db3035d2 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Tue, 6 May 2025 17:30:03 +0100 Subject: [PATCH 159/166] Remove `ObjectMessage.channel` See Simon's comment [1]. The `channel` property should not be expected in the wire protocol for object messages, just like it isn't present in a message or presencemessage. [1] https://github.com/ably/specification/pull/279#discussion_r1996035652 --- src/plugins/objects/objectmessage.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/plugins/objects/objectmessage.ts b/src/plugins/objects/objectmessage.ts index d02a7e99ce..83bb0a4b72 100644 --- a/src/plugins/objects/objectmessage.ts +++ b/src/plugins/objects/objectmessage.ts @@ -157,7 +157,6 @@ export class ObjectMessage { timestamp?: number; clientId?: string; connectionId?: string; - channel?: string; extras?: any; /** * Describes an operation to be applied to an object. @@ -499,7 +498,6 @@ export class ObjectMessage { if (this.timestamp) result += '; timestamp=' + this.timestamp; if (this.clientId) result += '; clientId=' + this.clientId; if (this.connectionId) result += '; connectionId=' + this.connectionId; - if (this.channel) result += '; channel=' + this.channel; // TODO: prettify output for operation and object and encode buffers. // see examples for data in Message and PresenceMessage if (this.operation) result += '; operation=' + JSON.stringify(this.operation); From 18a06844feab1240f30cee988da17a8d7c9e2449 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 7 May 2025 13:54:33 +0100 Subject: [PATCH 160/166] Handle `MapEntry.data` could be missing in the map entry `MapEntry.data` will not be set for tombstone properties in a map, this commit ensures we handle it correctly. --- src/plugins/objects/livemap.ts | 24 +++++---- src/plugins/objects/objectmessage.ts | 18 ++++--- test/realtime/objects.test.js | 75 +++++++++++++++++++++++++++- 3 files changed, 98 insertions(+), 19 deletions(-) diff --git a/src/plugins/objects/livemap.ts b/src/plugins/objects/livemap.ts index 1c9a3b58b0..b7525fc3e2 100644 --- a/src/plugins/objects/livemap.ts +++ b/src/plugins/objects/livemap.ts @@ -822,18 +822,20 @@ export class LiveMap extends LiveObject { - let liveData: ObjectData; + let liveData: ObjectData | undefined = undefined; - if (!this._client.Utils.isNil(entry.data.objectId)) { - liveData = { objectId: entry.data.objectId } as ObjectIdObjectData; - } else { - liveData = { - encoding: entry.data.encoding, - boolean: entry.data.boolean, - bytes: entry.data.bytes, - number: entry.data.number, - string: entry.data.string, - } as ValueObjectData; + if (!this._client.Utils.isNil(entry.data)) { + if (!this._client.Utils.isNil(entry.data.objectId)) { + liveData = { objectId: entry.data.objectId } as ObjectIdObjectData; + } else { + liveData = { + encoding: entry.data.encoding, + boolean: entry.data.boolean, + bytes: entry.data.bytes, + number: entry.data.number, + string: entry.data.string, + } as ValueObjectData; + } } const liveDataEntry: LiveMapEntry = { diff --git a/src/plugins/objects/objectmessage.ts b/src/plugins/objects/objectmessage.ts index 83bb0a4b72..d1cf1a93b8 100644 --- a/src/plugins/objects/objectmessage.ts +++ b/src/plugins/objects/objectmessage.ts @@ -67,7 +67,7 @@ export interface MapEntry { */ timeserial?: string; /** The data that represents the value of the map entry. */ - data: ObjectData; + data?: ObjectData; } /** An ObjectMap object represents a map of key-value pairs. */ @@ -321,7 +321,9 @@ export class ObjectMessage { format: Utils.Format | undefined, ): Promise { for (const entry of Object.values(mapEntries)) { - await ObjectMessage._decodeObjectData(entry.data, client, format); + if (entry.data) { + await ObjectMessage._decodeObjectData(entry.data, client, format); + } } } @@ -361,8 +363,10 @@ export class ObjectMessage { if (objectOperationCopy.map?.entries) { Object.entries(objectOperationCopy.map.entries).forEach(([key, entry]) => { - // use original "objectOperation" object when encoding values, so we have access to original buffer values. - entry.data = ObjectMessage._encodeObjectData(objectOperation?.map?.entries?.[key].data!, encodeObjectDataFn); + if (entry.data) { + // use original "objectOperation" object when encoding values, so we have access to original buffer values. + entry.data = ObjectMessage._encodeObjectData(objectOperation?.map?.entries?.[key].data!, encodeObjectDataFn); + } }); } @@ -386,8 +390,10 @@ export class ObjectMessage { if (objectStateCopy.map?.entries) { Object.entries(objectStateCopy.map.entries).forEach(([key, entry]) => { - // use original "objectState" object when encoding values, so we have access to original buffer values. - entry.data = ObjectMessage._encodeObjectData(objectState?.map?.entries?.[key].data!, encodeObjectDataFn); + if (entry.data) { + // use original "objectState" object when encoding values, so we have access to original buffer values. + entry.data = ObjectMessage._encodeObjectData(objectState?.map?.entries?.[key].data!, encodeObjectDataFn); + } }); } diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js index 5d60a81f43..7071de9b10 100644 --- a/test/realtime/objects.test.js +++ b/test/realtime/objects.test.js @@ -455,7 +455,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function const objectSyncSequenceScenarios = [ { allTransportsAndProtocols: true, - description: 'builds object tree from OBJECT_SYNC sequence on channel attachment', + description: 'OBJECT_SYNC sequence builds object tree on channel attachment', action: async (ctx) => { const { client } = ctx; @@ -506,6 +506,68 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, }, + { + allTransportsAndProtocols: true, + description: 'OBJECT_SYNC sequence builds object tree with all operations applied', + action: async (ctx) => { + const { root, objects, helper, clientOptions, channelName } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + + // MAP_CREATE + const map = await objects.createMap({ shouldStay: 'foo', shouldDelete: 'bar' }); + // COUNTER_CREATE + const counter = await objects.createCounter(1); + + await Promise.all([root.set('map', map), root.set('counter', counter), objectsCreatedPromise]); + + const operationsAppliedPromise = Promise.all([ + waitForMapKeyUpdate(map, 'anotherKey'), + waitForMapKeyUpdate(map, 'shouldDelete'), + waitForCounterUpdate(counter), + ]); + + await Promise.all([ + // MAP_SET + map.set('anotherKey', 'baz'), + // MAP_REMOVE + map.remove('shouldDelete'), + // COUNTER_INC + counter.increment(10), + operationsAppliedPromise, + ]); + + // create a new client and check it syncs with the aggregated data + const client2 = RealtimeWithObjects(helper, clientOptions); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const channel2 = client2.channels.get(channelName, channelOptionsWithObjects()); + const objects2 = channel2.objects; + + await channel2.attach(); + const root2 = await objects2.getRoot(); + + expect(root2.get('counter'), 'Check counter exists').to.exist; + expect(root2.get('counter').value()).to.equal(11, 'Check counter has correct value'); + + expect(root2.get('map'), 'Check map exists').to.exist; + expect(root2.get('map').size()).to.equal(2, 'Check map has correct number of keys'); + expect(root2.get('map').get('shouldStay')).to.equal( + 'foo', + 'Check map has correct value for "shouldStay" key', + ); + expect(root2.get('map').get('anotherKey')).to.equal( + 'baz', + 'Check map has correct value for "anotherKey" key', + ); + expect(root2.get('map').get('shouldDelete'), 'Check map does not have "shouldDelete" key').to.not.exist; + }, client2); + }, + }, + { allTransportsAndProtocols: true, description: 'LiveCounter is initialized with initial value from OBJECT_SYNC sequence', @@ -3576,7 +3638,16 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await channel.attach(); const root = await objects.getRoot(); - await scenario.action({ objects, root, objectsHelper, channelName, channel, client, helper }); + await scenario.action({ + objects, + root, + objectsHelper, + channelName, + channel, + client, + helper, + clientOptions, + }); }, client); }, ); From 882c59235f0e3ef0b9102f00e23e43f81d054b92 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 7 May 2025 15:20:13 +0100 Subject: [PATCH 161/166] Fix `ObjectMessage.encode` should not be async --- src/plugins/objects/objectmessage.ts | 2 +- test/realtime/objects.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/objects/objectmessage.ts b/src/plugins/objects/objectmessage.ts index d1cf1a93b8..70231178d3 100644 --- a/src/plugins/objects/objectmessage.ts +++ b/src/plugins/objects/objectmessage.ts @@ -186,7 +186,7 @@ export class ObjectMessage { * * Uses encoding functions from regular `Message` processing. */ - static async encode(message: ObjectMessage, messageEncoding: typeof MessageEncoding): Promise { + static encode(message: ObjectMessage, messageEncoding: typeof MessageEncoding): ObjectMessage { const encodeInitialValueFn: EncodeInitialValueFunction = (data, encoding) => { const { data: encodedData, encoding: newEncoding } = messageEncoding.encodeData(data, encoding); diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js index 7071de9b10..d3b88ab336 100644 --- a/test/realtime/objects.test.js +++ b/test/realtime/objects.test.js @@ -4852,7 +4852,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function /** @nospec */ forScenarios(this, objectMessageSizeScenarios, function (helper, scenario) { helper.recordPrivateApi('call.ObjectMessage.encode'); - ObjectsPlugin.ObjectMessage.encode(scenario.message); + ObjectsPlugin.ObjectMessage.encode(scenario.message, MessageEncoding); helper.recordPrivateApi('call.BufferUtils.utf8Encode'); // was called by a scenario to create buffers helper.recordPrivateApi('call.ObjectMessage.fromValues'); // was called by a scenario to create an ObjectMessage instance helper.recordPrivateApi('call.Utils.dataSizeBytes'); // was called by a scenario to calculated the expected byte size From d565623d6c1dde7c4e9408794b99a90e1df660c5 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 7 May 2025 15:21:03 +0100 Subject: [PATCH 162/166] Add nonce and initial value ObjectMessage message size tests --- test/realtime/objects.test.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js index d3b88ab336..bccafb959e 100644 --- a/test/realtime/objects.test.js +++ b/test/realtime/objects.test.js @@ -4621,6 +4621,23 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }), expected: 0, }, + { + description: 'nonce', + message: objectMessageFromValues({ + operation: { nonce: '1234567890' }, + }), + expected: 0, + }, + { + description: 'initial value', + message: objectMessageFromValues({ + operation: { + initialValue: BufferUtils.utf8Encode('{"counter":{"count":1}}'), + initialValueEncoding: 'json', + }, + }), + expected: 0, + }, { description: 'map create op no payload', message: objectMessageFromValues({ From d2f79e08583f25bd94d4658ab3664f0562d3f7b1 Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Wed, 7 May 2025 15:43:05 +0100 Subject: [PATCH 163/166] Fix "Data type is unsupported" errors for Object message encoding --- src/common/lib/types/basemessage.ts | 16 ++++++++++------ src/plugins/objects/objectmessage.ts | 18 +++++++++++++++--- src/plugins/objects/objects.ts | 2 +- test/realtime/objects.test.js | 3 ++- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/common/lib/types/basemessage.ts b/src/common/lib/types/basemessage.ts index 98f8f473ad..b3cfc69353 100644 --- a/src/common/lib/types/basemessage.ts +++ b/src/common/lib/types/basemessage.ts @@ -95,7 +95,14 @@ export async function encryptData( * Implements RSL4 and RSL5. */ export async function encode(msg: T, options: unknown): Promise { - const { data, encoding } = encodeData(msg.data, msg.encoding); + // RSL4a, supported types + const isNativeDataType = + typeof msg.data == 'string' || + Platform.BufferUtils.isBuffer(msg.data) || + msg.data === null || + msg.data === undefined; + const { data, encoding } = encodeData(msg.data, msg.encoding, isNativeDataType); + msg.data = data; msg.encoding = encoding; @@ -109,12 +116,9 @@ export async function encode(msg: T, options: unknown): P export function encodeData( data: any, encoding: string | null | undefined, + isNativeDataType: boolean, ): { data: any; encoding: string | null | undefined } { - // RSL4a, supported types - const nativeDataType = - typeof data == 'string' || Platform.BufferUtils.isBuffer(data) || data === null || data === undefined; - - if (nativeDataType) { + if (isNativeDataType) { // nothing to do with the native data types at this point return { data, diff --git a/src/plugins/objects/objectmessage.ts b/src/plugins/objects/objectmessage.ts index 70231178d3..f5f2abe4e4 100644 --- a/src/plugins/objects/objectmessage.ts +++ b/src/plugins/objects/objectmessage.ts @@ -186,9 +186,21 @@ export class ObjectMessage { * * Uses encoding functions from regular `Message` processing. */ - static encode(message: ObjectMessage, messageEncoding: typeof MessageEncoding): ObjectMessage { + static encode(message: ObjectMessage, client: BaseClient): ObjectMessage { const encodeInitialValueFn: EncodeInitialValueFunction = (data, encoding) => { - const { data: encodedData, encoding: newEncoding } = messageEncoding.encodeData(data, encoding); + const isNativeDataType = + typeof data == 'string' || + typeof data == 'number' || + typeof data == 'boolean' || + client.Platform.BufferUtils.isBuffer(data) || + data === null || + data === undefined; + + const { data: encodedData, encoding: newEncoding } = client.MessageEncoding.encodeData( + data, + encoding, + isNativeDataType, + ); return { data: encodedData, @@ -291,7 +303,7 @@ export class ObjectMessage { // initial value object may contain user provided data that requires an additional encoding (for example buffers as map keys). // so we need to encode that data first as if we were sending it over the wire. we can use an ObjectMessage methods for this const msg = ObjectMessage.fromValues({ operation: initialValue }, client.Utils, client.MessageEncoding); - ObjectMessage.encode(msg, client.MessageEncoding); + ObjectMessage.encode(msg, client); const { operation: initialValueWithDataEncoding } = ObjectMessage._encodeForWireProtocol( msg, client.MessageEncoding, diff --git a/src/plugins/objects/objects.ts b/src/plugins/objects/objects.ts index 24ea5d5ab1..a5b82c78bc 100644 --- a/src/plugins/objects/objects.ts +++ b/src/plugins/objects/objects.ts @@ -296,7 +296,7 @@ export class Objects { async publish(objectMessages: ObjectMessage[]): Promise { this._channel.throwIfUnpublishableState(); - objectMessages.forEach((x) => ObjectMessage.encode(x, this._client.MessageEncoding)); + objectMessages.forEach((x) => ObjectMessage.encode(x, this._client)); const maxMessageSize = this._client.options.maxMessageSize; const size = objectMessages.reduce((acc, msg) => acc + msg.getMessageSize(), 0); if (size > maxMessageSize) { diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js index bccafb959e..55c3eea126 100644 --- a/test/realtime/objects.test.js +++ b/test/realtime/objects.test.js @@ -4868,8 +4868,9 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function /** @nospec */ forScenarios(this, objectMessageSizeScenarios, function (helper, scenario) { + const client = RealtimeWithObjects(helper, { autoConnect: false }); helper.recordPrivateApi('call.ObjectMessage.encode'); - ObjectsPlugin.ObjectMessage.encode(scenario.message, MessageEncoding); + ObjectsPlugin.ObjectMessage.encode(scenario.message, client); helper.recordPrivateApi('call.BufferUtils.utf8Encode'); // was called by a scenario to create buffers helper.recordPrivateApi('call.ObjectMessage.fromValues'); // was called by a scenario to create an ObjectMessage instance helper.recordPrivateApi('call.Utils.dataSizeBytes'); // was called by a scenario to calculated the expected byte size From b777e50069f9bc87c671857be5d88716bf15446c Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 8 May 2025 17:01:54 +0100 Subject: [PATCH 164/166] Fix object references were reset on channel detach/fail and reattach with HAS_OBJECTS=false Object references must be kept the same whenever possible so that end-user keeps the correct reference and its subscriptions to the object whenever resynchronization happens, see [1]. On channel DETACHED or FAILED only the objects data should be removed, and no update events should be emitted, see [2]. [1] https://ably.atlassian.net/wiki/spaces/LOB/pages/3382738945/LODR-022+Realtime+Client+read-only+internal+spec#7.-Decoupling-of-the-underlying-state-data-and-LiveObject-class-instances [2] https://ably.atlassian.net/wiki/spaces/LOB/pages/3784933378/LODR-032+Realtime+Client+behavior+under+channel+states --- src/plugins/objects/liveobject.ts | 19 ++++++-- src/plugins/objects/objects.ts | 14 +++--- src/plugins/objects/objectspool.ts | 30 ++++++++++-- src/plugins/objects/syncobjectsdatapool.ts | 2 +- test/realtime/objects.test.js | 54 ++++++++++++++++++++++ 5 files changed, 103 insertions(+), 16 deletions(-) diff --git a/src/plugins/objects/liveobject.ts b/src/plugins/objects/liveobject.ts index 3a12114547..d0870d5028 100644 --- a/src/plugins/objects/liveobject.ts +++ b/src/plugins/objects/liveobject.ts @@ -159,11 +159,13 @@ export abstract class LiveObject< * * @internal */ - tombstone(): void { + tombstone(): TUpdate { this._tombstone = true; this._tombstonedAt = Date.now(); - this._dataRef = this._getZeroValueData(); + const update = this.clearData(); this._lifecycleEvents.emit(LiveObjectLifecycleEvent.deleted); + + return update; } /** @@ -180,6 +182,15 @@ export abstract class LiveObject< return this._tombstonedAt; } + /** + * @internal + */ + clearData(): TUpdate { + const previousDataRef = this._dataRef; + this._dataRef = this._getZeroValueData(); + return this._updateFromDataDiff(previousDataRef, this._dataRef); + } + /** * Returns true if the given serial indicates that the operation to which it belongs should be applied to the object. * @@ -200,9 +211,7 @@ export abstract class LiveObject< } protected _applyObjectDelete(): TUpdate { - const previousDataRef = this._dataRef; - this.tombstone(); - return this._updateFromDataDiff(previousDataRef, this._dataRef); + return this.tombstone(); } /** diff --git a/src/plugins/objects/objects.ts b/src/plugins/objects/objects.ts index a5b82c78bc..2cad0d685a 100644 --- a/src/plugins/objects/objects.ts +++ b/src/plugins/objects/objects.ts @@ -265,8 +265,9 @@ export class Objects { if (!hasObjects) { // if no HAS_OBJECTS flag received on attach, we can end sync sequence immediately and treat it as no objects on a channel. - this._objectsPool.reset(); - this._syncObjectsDataPool.reset(); + // reset the objects pool to its initial state, and emit update events so subscribers to root object get notified about changes. + this._objectsPool.resetToInitialPool(true); + this._syncObjectsDataPool.clear(); // defer the state change event until the next tick if we started a new sequence just now due to being in initialized state. // this allows any event listeners to process the start of the new sequence event that was emitted earlier during this event loop. this._endSync(fromInitializedState); @@ -284,8 +285,9 @@ export class Objects { case 'detached': case 'failed': - this._objectsPool.reset(); - this._syncObjectsDataPool.reset(); + // do not emit data update events as the actual current state of Objects data is unknown when we're in these channel states + this._objectsPool.clearObjectsData(false); + this._syncObjectsDataPool.clear(); break; } } @@ -329,7 +331,7 @@ export class Objects { private _startNewSync(syncId?: string, syncCursor?: string): void { // need to discard all buffered object operation messages on new sync start this._bufferedObjectOperations = []; - this._syncObjectsDataPool.reset(); + this._syncObjectsDataPool.clear(); this._currentSyncId = syncId; this._currentSyncCursor = syncCursor; this._stateChange(ObjectsState.syncing, false); @@ -342,7 +344,7 @@ export class Objects { this._applyObjectMessages(this._bufferedObjectOperations); this._bufferedObjectOperations = []; - this._syncObjectsDataPool.reset(); + this._syncObjectsDataPool.clear(); this._currentSyncId = undefined; this._currentSyncCursor = undefined; this._stateChange(ObjectsState.synced, deferStateEvent); diff --git a/src/plugins/objects/objectspool.ts b/src/plugins/objects/objectspool.ts index 30d7c43d1b..99a9a1f1f1 100644 --- a/src/plugins/objects/objectspool.ts +++ b/src/plugins/objects/objectspool.ts @@ -18,7 +18,7 @@ export class ObjectsPool { constructor(private _objects: Objects) { this._client = this._objects.getClient(); - this._pool = this._getInitialPool(); + this._pool = this._createInitialPool(); this._gcInterval = setInterval(() => { this._onGCInterval(); }, DEFAULTS.gcInterval); @@ -44,8 +44,30 @@ export class ObjectsPool { this._pool.set(objectId, liveObject); } - reset(): void { - this._pool = this._getInitialPool(); + /** + * Removes all objects but root from the pool and clears the data for root. + * Does not create a new root object, so the reference to the root object remains the same. + */ + resetToInitialPool(emitUpdateEvents: boolean): void { + // clear the pool first and keep the root object + const root = this._pool.get(ROOT_OBJECT_ID)!; + this._pool.clear(); + this._pool.set(root.getObjectId(), root); + + // clear the data, this will only clear the root object + this.clearObjectsData(emitUpdateEvents); + } + + /** + * Clears the data stored for all objects in the pool. + */ + clearObjectsData(emitUpdateEvents: boolean): void { + for (const object of this._pool.values()) { + const update = object.clearData(); + if (emitUpdateEvents) { + object.notifyUpdated(update); + } + } } createZeroValueObjectIfNotExists(objectId: string): LiveObject { @@ -71,7 +93,7 @@ export class ObjectsPool { return zeroValueObject; } - private _getInitialPool(): Map { + private _createInitialPool(): Map { const pool = new Map(); const root = LiveMap.zeroValue(this._objects, ROOT_OBJECT_ID); pool.set(root.getObjectId(), root); diff --git a/src/plugins/objects/syncobjectsdatapool.ts b/src/plugins/objects/syncobjectsdatapool.ts index bb069adbe2..e411b502a5 100644 --- a/src/plugins/objects/syncobjectsdatapool.ts +++ b/src/plugins/objects/syncobjectsdatapool.ts @@ -45,7 +45,7 @@ export class SyncObjectsDataPool { return this._pool.size === 0; } - reset(): void { + clear(): void { this._pool.clear(); } diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js index 55c3eea126..d2f36fc33f 100644 --- a/test/realtime/objects.test.js +++ b/test/realtime/objects.test.js @@ -135,6 +135,30 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); } + async function waitForObjectSync(helper, client) { + return new Promise((resolve, reject) => { + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + const transport = client.connection.connectionManager.activeProtocol.getTransport(); + const onProtocolMessageOriginal = transport.onProtocolMessage; + + helper.recordPrivateApi('replace.transport.onProtocolMessage'); + transport.onProtocolMessage = function (message) { + try { + helper.recordPrivateApi('call.transport.onProtocolMessage'); + onProtocolMessageOriginal.call(transport, message); + + if (message.action === 20) { + helper.recordPrivateApi('replace.transport.onProtocolMessage'); + transport.onProtocolMessage = onProtocolMessageOriginal; + resolve(); + } + } catch (err) { + reject(err); + } + }; + }); + } + /** * The channel with fixture data may not yet be populated by REST API requests made by ObjectsHelper. * This function waits for a channel to have all keys set. @@ -568,6 +592,36 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }, }, + { + description: 'OBJECT_SYNC sequence does not change references to existing objects', + action: async (ctx) => { + const { root, objects, helper, channel } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + + const map = await objects.createMap(); + const counter = await objects.createCounter(); + await Promise.all([root.set('map', map), root.set('counter', counter), objectsCreatedPromise]); + await channel.detach(); + + // wait for the actual OBJECT_SYNC message to confirm it was received and processed + const objectSyncPromise = waitForObjectSync(helper, channel.client); + await channel.attach(); + await objectSyncPromise; + + const newRootRef = await channel.objects.getRoot(); + const newMapRef = newRootRef.get('map'); + const newCounterRef = newRootRef.get('counter'); + + expect(newRootRef).to.equal(root, 'Check root reference is the same after OBJECT_SYNC sequence'); + expect(newMapRef).to.equal(map, 'Check map reference is the same after OBJECT_SYNC sequence'); + expect(newCounterRef).to.equal(counter, 'Check counter reference is the same after OBJECT_SYNC sequence'); + }, + }, + { allTransportsAndProtocols: true, description: 'LiveCounter is initialized with initial value from OBJECT_SYNC sequence', From 671243c24c872e0d1ebf43b38f9ddee51d657cbe Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 8 May 2025 17:09:00 +0100 Subject: [PATCH 165/166] Remove `enableChannelState` feature flag --- test/common/modules/testapp_manager.js | 1 - test/package/browser/template/src/index-objects.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/test/common/modules/testapp_manager.js b/test/common/modules/testapp_manager.js index 7852d79f19..0f4b7fa9dc 100644 --- a/test/common/modules/testapp_manager.js +++ b/test/common/modules/testapp_manager.js @@ -133,7 +133,6 @@ define(['globals', 'ably'], function (ablyGlobals, ably) { callback(err); return; } - testData.post_apps.featureFlags = ['enableChannelState']; var postData = JSON.stringify(testData.post_apps); var postOptions = { host: restHost, diff --git a/test/package/browser/template/src/index-objects.ts b/test/package/browser/template/src/index-objects.ts index 7dbca53573..ada25ba889 100644 --- a/test/package/browser/template/src/index-objects.ts +++ b/test/package/browser/template/src/index-objects.ts @@ -33,7 +33,7 @@ type ExplicitRootType = { }; globalThis.testAblyPackage = async function () { - const key = await createSandboxAblyAPIKey({ featureFlags: ['enableChannelState'] }); + const key = await createSandboxAblyAPIKey(); const realtime = new Ably.Realtime({ key, environment: 'sandbox', plugins: { Objects } }); From a238857b541fa81a1f7465a95dff2f655ad3324f Mon Sep 17 00:00:00 2001 From: Andrew Bulat Date: Thu, 8 May 2025 17:41:12 +0100 Subject: [PATCH 166/166] Objects Write API should throw an error when "echoMessages" client option is set to false --- src/plugins/objects/objects.ts | 11 +++++++++ test/common/modules/private_api_recorder.js | 1 + test/realtime/objects.test.js | 26 ++++++++++++++++++--- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/plugins/objects/objects.ts b/src/plugins/objects/objects.ts index 2cad0d685a..45223cee8e 100644 --- a/src/plugins/objects/objects.ts +++ b/src/plugins/objects/objects.ts @@ -326,6 +326,7 @@ export class Objects { throwIfInvalidWriteApiConfiguration(): void { this._throwIfMissingChannelMode('object_publish'); this._throwIfInChannelState(['detached', 'failed', 'suspended']); + this._throwIfEchoMessagesDisabled(); } private _startNewSync(syncId?: string, syncCursor?: string): void { @@ -494,4 +495,14 @@ export class Objects { throw this._client.ErrorInfo.fromValues(this._channel.invalidStateError()); } } + + private _throwIfEchoMessagesDisabled(): void { + if (this._channel.client.options.echoMessages === false) { + throw new this._channel.client.ErrorInfo( + `"echoMessages" client option must be enabled for this operation`, + 40000, + 400, + ); + } + } } diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index 86e6d099f8..cecf6670b8 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -156,6 +156,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'write.connectionManager.lastActivity', 'write.connectionManager.msgSerial', 'write.connectionManager.wsHosts', + 'write.realtime.options.echoMessages', 'write.realtime.options.realtimeHost', 'write.realtime.options.wsConnectivityCheckUrl', 'write.realtime.options.timeouts.realtimeRequestTimeout', diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js index d2f36fc33f..220694199c 100644 --- a/test/realtime/objects.test.js +++ b/test/realtime/objects.test.js @@ -4450,7 +4450,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function expect(() => map.remove()).to.throw(errorMsg); }; - const channelConfigurationScenarios = [ + const clientConfigurationScenarios = [ { description: 'public API throws missing object modes error when attached without correct modes', action: async (ctx) => { @@ -4574,10 +4574,30 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function }); }, }, + + { + description: 'public write API throws invalid channel option when "echoMessages" is disabled', + action: async (ctx) => { + const { objects, client, map, counter, helper } = ctx; + + // obtain batch context with valid client options first + await objects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate echoMessages was disabled + helper.recordPrivateApi('write.realtime.options.echoMessages'); + client.options.echoMessages = false; + + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: '"echoMessages" client option' }); + }); + + await expectWriteApiToThrow({ objects, map, counter, errorMsg: '"echoMessages" client option' }); + }, + }, ]; /** @nospec */ - forScenarios(this, channelConfigurationScenarios, async function (helper, scenario, clientOptions, channelName) { + forScenarios(this, clientConfigurationScenarios, async function (helper, scenario, clientOptions, channelName) { const objectsHelper = new ObjectsHelper(helper); const client = RealtimeWithObjects(helper, clientOptions); @@ -4600,7 +4620,7 @@ define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function await root.set('counter', counter); await objectsCreatedPromise; - await scenario.action({ objects, objectsHelper, channelName, channel, root, map, counter, helper }); + await scenario.action({ objects, objectsHelper, channelName, channel, root, map, counter, helper, client }); }, client); });