diff --git a/Gruntfile.js b/Gruntfile.js index bafac9f510..7ba8985289 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -73,7 +73,7 @@ module.exports = function (grunt) { }); }); - grunt.registerTask('build', ['webpack:all', 'build:browser', 'build:node', 'build:push']); + grunt.registerTask('build', ['webpack:all', 'build:browser', 'build:node', 'build:push', 'build:objects']); grunt.registerTask('all', ['build', 'requirejs']); @@ -138,9 +138,26 @@ module.exports = function (grunt) { }); }); + grunt.registerTask('build:objects', function () { + var done = this.async(); + + Promise.all([ + esbuild.build(esbuildConfig.objectsPluginConfig), + esbuild.build(esbuildConfig.objectsPluginCdnConfig), + esbuild.build(esbuildConfig.minifiedObjectsPluginCdnConfig), + ]) + .then(() => { + done(true); + }) + .catch((err) => { + done(err); + }); + }); + grunt.registerTask('test:webserver', 'Launch the Mocha test web server on http://localhost:3000/', [ 'build:browser', 'build:push', + 'build:objects', 'checkGitSubmodules', 'mocha:webserver', ]); diff --git a/README.md b/README.md index 7243233085..f62b9a8f12 100644 --- a/README.md +++ b/README.md @@ -584,7 +584,333 @@ const client = new Ably.Rest({ The Push plugin is developed as part of the Ably client library, so it is available for the same versions as the Ably client library itself. It also means that it follows the same semantic versioning rules as they were defined for [the Ably client library](#for-browsers). For example, to lock into a major or minor version of the Push plugin, you can specify a specific version number such as https://cdn.ably.com/lib/push.umd.min-2.js for all v2._ versions, or https://cdn.ably.com/lib/push.umd.min-2.4.js for all v2.4._ versions, or you can lock into a single release with https://cdn.ably.com/lib/push.umd.min-2.4.0.js. Note you can load the non-minified version by omitting `.min` from the URL such as https://cdn.ably.com/lib/push.umd-2.js. -For more information on publishing push notifcations over Ably, see the [Ably push documentation](https://ably.com/docs/push). +For more information on publishing push notifications over Ably, see the [Ably push documentation](https://ably.com/docs/push). + +### LiveObjects + +#### Using the Objects plugin + +LiveObjects functionality is supported for Realtime clients via the Objects plugin. In order to use Objects on a channel, you must pass in the plugin via client options. + +```typescript +import * as Ably from 'ably'; +import Objects from 'ably/objects'; + +const client = new Ably.Realtime({ + ...options, + plugins: { Objects }, +}); +``` + +Objects plugin also works with the [Modular variant](#modular-tree-shakable-variant) of the library. + +Alternatively, you can load the Objects plugin directly in your HTML using `script` tag (in case you can't use a package manager): + +```html + +``` + +When loaded this way, the Objects plugin will be available on the global object via the `AblyObjectsPlugin` property, so you will need to pass it to the Ably instance as follows: + +```typescript +const client = new Ably.Realtime({ + ...options, + plugins: { Objects: AblyObjectsPlugin }, +}); +``` + +The Objects plugin is developed as part of the Ably client library, so it is available for the same versions as the Ably client library itself. It also means that it follows the same semantic versioning rules as they were defined for [the Ably client library](#for-browsers). For example, to lock into a major or minor version of the Objects plugin, you can specify a specific version number such as https://cdn.ably.com/lib/objects.umd.min-2.js for all v2._ versions, or https://cdn.ably.com/lib/objects.umd.min-2.4.js for all v2.4._ versions, or you can lock into a single release with https://cdn.ably.com/lib/objects.umd.min-2.4.0.js. Note you can load the non-minified version by omitting `.min` from the URL such as https://cdn.ably.com/lib/objects.umd-2.js. + +For more information about the LiveObjects product, see the [Ably LiveObjects documentation](https://ably.com/docs/liveobjects). + +#### Objects Channel Modes + +To use the Objects on a channel, clients must attach to a channel with the correct channel mode: + +- `object_subscribe` - required to retrieve Objects for a channel +- `object_publish` - required to create new and modify existing Objects on a channel + +```typescript +const client = new Ably.Realtime({ + // authentication options + ...options, + plugins: { Objects }, +}); +const channelOptions = { modes: ['object_subscribe', 'object_publish'] }; +const channel = client.channels.get('my_objects_channel', channelOptions); +const objects = channel.objects; +``` + +The authentication token must include corresponding capabilities for the client to interact with Objects. + +#### Getting the Root Object + +The root object represents the top-level entry point for objects within a channel. It gives access to all other nested objects. + +```typescript +const root = await objects.getRoot(); +``` + +The root object is a `LiveMap` instance and serves as the starting point for storing and organizing Objects on a channel. + +#### Object Types + +LiveObjects currently supports two primary data structures; `LiveMap` and `LiveCounter`. + +`LiveMap` - A key/value map data structure, similar to a JavaScript `Map`, where all changes are synchronized across clients in realtime. It enables you to store primitive values and other objects, enabling composability. + +You can use `LiveMap` as follows: + +```typescript +// root object is a LiveMap +const root = await objects.getRoot(); + +// you can read values for a key with .get +root.get('foo'); +root.get('bar'); + +// get a number of key/value pairs in a map with .size +root.size(); + +// iterate over keys/values in a map +for (const [key, value] of root.entries()) { + /**/ +} +for (const key of root.keys()) { + /**/ +} +for (const value of root.values()) { + /**/ +} + +// set keys on a map with .set +// different data types are supported +await root.set('foo', 'Alice'); +await root.set('bar', 1); +await root.set('baz', true); +await root.set('qux', new Uint8Array([21, 31])); +// as well as other objects +const counter = await objects.createCounter(); +await root.set('quux', counter); + +// and you can remove keys with .remove +await root.remove('name'); +``` + +`LiveCounter` - A counter that can be incremented or decremented and is synchronized across clients in realtime + +You can use `LiveCounter` as follows: + +```typescript +const counter = await objects.createCounter(); + +// you can get current value of a counter with .value +counter.value(); + +// and change its value with .increment or .decrement +await counter.increment(5); +await counter.decrement(2); +``` + +#### Subscribing to Updates + +Subscribing to updates on objects enables you to receive changes made by other clients in realtime. Since multiple clients may modify the same objects, subscribing ensures that your application reacts to external updates as soon as they are received. + +Additionally, mutation methods such as `LiveMap.set`, `LiveCounter.increment`, and `LiveCounter.decrement` do not directly edit the current state of the object locally. Instead, they send the intended operation to the Ably system, and the change is applied to the local object only when the corresponding realtime operation is echoed back to the client. This means that the state you retrieve immediately after a mutation may not reflect the latest updates yet. + +You can subscribe to updates on all objects using subscription listeners as follows: + +```typescript +const root = await objects.getRoot(); + +// subscribe to updates on a LiveMap +root.subscribe((update: LiveMapUpdate) => { + console.log('LiveMap "name" key:', root.get('name')); // can read the current value for a key in a map inside this callback + console.log('LiveMap update details:', update); // and can get update details from the provided update object +}); + +// subscribe to updates on a LiveCounter +const counter = await objects.createCounter(); +counter.subscribe((update: LiveCounterUpdate) => { + console.log('LiveCounter new value:', counter.value()); // can read the current value of the counter inside this callback + console.log('LiveCounter update details:', update); // and can get update details from the provided update object +}); + +// perform operations on LiveMap and LiveCounter +await root.set('name', 'Alice'); +// LiveMap "name" key: Alice +// LiveMap update details: { update: { name: 'updated' } } + +await root.remove('name'); +// LiveMap "name" key: undefined +// LiveMap update details: { update: { name: 'removed' } } + +await counter.increment(5); +// LiveCounter new value: 5 +// LiveCounter update details: { update: { amount: 5 } } + +await counter.decrement(2); +// LiveCounter new value: 3 +// LiveCounter update details: { update: { amount: -2 } } +``` + +You can deregister subscription listeners as follows: + +```typescript +// use dedicated unsubscribe function from the .subscribe call +const { unsubscribe } = root.subscribe(() => {}); +unsubscribe(); + +// call .unsubscribe with a listener reference +const listener = () => {}; +root.subscribe(listener); +root.unsubscribe(listener); + +// deregister all listeners using .unsubscribeAll +root.unsubscribeAll(); +``` + +#### Creating New Objects + +New `LiveMap` and `LiveCounter` objects can be created as follows: + +```typescript +const counter = await objects.createCounter(123); // with optional initial counter value +const map = await objects.createMap({ key: 'value' }); // with optional initial map entries +``` + +To persist them on a channel and share them between clients, they must be assigned to a parent `LiveMap` that is connected to the root object through the object hierarchy: + +```typescript +const root = await objects.getRoot(); + +const counter = await objects.createCounter(); +const map = await objects.createMap({ counter }); +const outerMap = await objects.createMap({ map }); + +await root.set('outerMap', outerMap); + +// resulting structure: +// root (LiveMap) +// └── outerMap (LiveMap) +// └── map (LiveMap) +// └── counter (LiveCounter) +``` + +#### Batch Operations + +Batching enables multiple operations to be grouped into a single channel message that is sent to the Ably service. This guarantees that all changes are applied atomically. + +Within a batch callback, the `BatchContext` instance provides wrapper objects around regular `LiveMap` and `LiveCounter` objects with a synchronous API for storing changes in the batch context. + +```typescript +await objects.batch((ctx) => { + const root = ctx.getRoot(); + + root.set('foo', 'bar'); + root.set('baz', 42); + + const counter = root.get('counter'); + counter.increment(5); + + // batched operations are sent to the Ably service when the batch callback returns +}); +``` + +#### Lifecycle Events + +LiveObjects emit events that allow you to monitor objects' lifecycle changes, such as synchronization progress and object deletions. + +**Synchronization Events** - the `syncing` and `synced` events notify when the local Objects state on a client is being synchronized with the Ably service. These events can be useful for displaying loading indicators, preventing user edits during synchronization, or triggering application logic when the data was loaded for the first time. + +```typescript +objects.on('syncing', () => { + console.log('Objects are syncing...'); + // Example: Show a loading indicator +}); + +objects.on('synced', () => { + console.log('Objects have been synced.'); + // Example: Hide loading indicator +}); +``` + +**Object Deletion Events** - objects that have been orphaned for a long period (i.e., not connected to the object tree by being set as a key in a map accessible from the root map object) will eventually be deleted. Once an object is deleted, it can no longer be interacted with. You should avoid accessing its data or trying to update its value and you should remove all references to the deleted object in your application. + +```typescript +const root = await objects.getRoot(); +const counter = root.get('counter'); + +counter.on('deleted', () => { + console.log('Object has been deleted.'); + // Example: Remove references to the object from the application +}); +``` + +To unsubscribe from lifecycle events: + +```typescript +// same API for channel.objects and LiveObject instances +// use dedicated off function from the .on call +const { off } = objects.on('synced', () => {}); +off(); + +// call .off with an event name and a listener reference +const listener = () => {}; +objects.on('synced', listener); +objects.off('synced', listener); + +// deregister all listeners using .offAll +objects.offAll(); +``` + +#### Typing Objects + +You can provide your own TypeScript typings for Objects by providing a globally defined `AblyObjectsTypes` interface. + +```typescript +// file: ably.config.d.ts +import { LiveCounter, LiveMap } from 'ably'; + +type MyCustomRoot = { + map: LiveMap<{ + foo: string; + counter: LiveCounter; + }>; +}; + +declare global { + export interface AblyObjectsTypes { + root: MyCustomRoot; + } +} +``` + +Note that using TypeScript typings for Objects does not provide runtime type checking; instead, it enables code completion and editor hints (if supported by your IDE) when interacting with the Objects API: + +```typescript +const root = await objects.getRoot(); // uses types defined by global AblyObjectsTypes interface by default + +const map = root.get('map'); // LiveMap<{ foo: string; counter: LiveCounter }> +map.set('foo', 1); // TypeError +map.get('counter').value(); // autocompletion for counter method names +``` + +You can also provide typings for the channel Objects when calling the `objects.getRoot` method, allowing you to have different typings for different channels: + +```typescript +type ReactionsRoot = { + hearts: LiveCounter; + likes: LiveCounter; +}; + +type PollsRoot = { + currentPoll: LiveMap; +}; + +const reactionsRoot = await reactionsChannel.objects.getRoot(); +const pollsRoot = await pollsChannel.objects.getRoot(); +``` ## Delta Plugin diff --git a/ably.d.ts b/ably.d.ts index 8a642f8b67..df8abdcef4 100644 --- a/ably.d.ts +++ b/ably.d.ts @@ -623,6 +623,11 @@ export interface CorePlugins { * A plugin which allows the client to be the target of push notifications. */ Push?: unknown; + + /** + * A plugin which allows the client to use LiveObjects functionality at {@link RealtimeChannel.objects}. + */ + Objects?: unknown; } /** @@ -871,11 +876,19 @@ declare namespace ChannelModes { */ type PRESENCE_SUBSCRIBE = 'PRESENCE_SUBSCRIBE' | 'presence_subscribe'; /** - * The client can publish annotations + * The client can publish object messages. + */ + type OBJECT_PUBLISH = 'OBJECT_PUBLISH' | 'object_publish'; + /** + * The client will receive object messages. + */ + type OBJECT_SUBSCRIBE = 'OBJECT_SUBSCRIBE' | 'object_subscribe'; + /** + * The client can publish annotations. */ type ANNOTATION_PUBLISH = 'ANNOTATION_PUBLISH' | 'annotation_publish'; /** - * The client will receive annotations + * The client will receive annotations. */ type ANNOTATION_SUBSCRIBE = 'ANNOTATION_SUBSCRIBE' | 'annotation_subscribe'; } @@ -890,6 +903,8 @@ export type ChannelMode = | ChannelModes.SUBSCRIBE | ChannelModes.PRESENCE | ChannelModes.PRESENCE_SUBSCRIBE + | ChannelModes.OBJECT_PUBLISH + | ChannelModes.OBJECT_SUBSCRIBE | ChannelModes.ANNOTATION_PUBLISH | ChannelModes.ANNOTATION_SUBSCRIBE; @@ -914,11 +929,19 @@ declare namespace ResolvedChannelModes { */ type PRESENCE_SUBSCRIBE = 'presence_subscribe'; /** - * The client can publish annotations + * The client can publish object messages. + */ + type OBJECT_PUBLISH = 'object_publish'; + /** + * The client will receive object messages. + */ + type OBJECT_SUBSCRIBE = 'object_subscribe'; + /** + * The client can publish annotations. */ type ANNOTATION_PUBLISH = 'annotation_publish'; /** - * The client will receive annotations + * The client will receive annotations. */ type ANNOTATION_SUBSCRIBE = 'annotation_subscribe'; } @@ -935,6 +958,8 @@ export type ResolvedChannelMode = | ResolvedChannelModes.SUBSCRIBE | ResolvedChannelModes.PRESENCE | ResolvedChannelModes.PRESENCE_SUBSCRIBE + | ResolvedChannelModes.OBJECT_PUBLISH + | ResolvedChannelModes.OBJECT_SUBSCRIBE | ResolvedChannelModes.ANNOTATION_PUBLISH | ResolvedChannelModes.ANNOTATION_SUBSCRIBE; @@ -1619,6 +1644,32 @@ export type DeregisterCallback = (device: DeviceDetails, callback: StandardCallb */ export type ErrorCallback = (error: ErrorInfo | null) => void; +/** + * A callback used in {@link LiveObject} to listen for updates to the object. + * + * @param update - The update object describing the changes made to the object. + */ +export type LiveObjectUpdateCallback = (update: T) => void; + +/** + * The callback used for the events emitted by {@link Objects}. + */ +export type ObjectsEventCallback = () => void; + +/** + * The callback used for the lifecycle events emitted by {@link LiveObject}. + */ +export type LiveObjectLifecycleEventCallback = () => void; + +/** + * A function passed to {@link Objects.batch} to group multiple Objects operations into a single channel message. + * + * Must not be `async`. + * + * @param batchContext - A {@link BatchContext} object that allows grouping Objects operations for this batch. + */ +export type BatchCallback = (batchContext: BatchContext) => void; + // Internal Interfaces // To allow a uniform (callback) interface between on and once even in the @@ -2185,6 +2236,499 @@ export declare interface PushChannel { listSubscriptions(params?: Record): Promise>; } +/** + * The `ObjectsEvents` namespace describes the possible values of the {@link ObjectsEvent} type. + */ +declare namespace ObjectsEvents { + /** + * The local copy of Objects on a channel is currently being synchronized with the Ably service. + */ + type SYNCING = 'syncing'; + /** + * The local copy of Objects on a channel has been synchronized with the Ably service. + */ + type SYNCED = 'synced'; +} + +/** + * Describes the events emitted by a {@link Objects} object. + */ +export type ObjectsEvent = ObjectsEvents.SYNCED | ObjectsEvents.SYNCING; + +/** + * The `LiveObjectLifecycleEvents` namespace describes the possible values of the {@link LiveObjectLifecycleEvent} type. + */ +declare namespace LiveObjectLifecycleEvents { + /** + * Indicates that the object has been deleted from the Objects pool and should no longer be interacted with. + */ + type DELETED = 'deleted'; +} + +/** + * Describes the events emitted by a {@link LiveObject} object. + */ +export type LiveObjectLifecycleEvent = LiveObjectLifecycleEvents.DELETED; + +/** + * Enables the Objects to be read, modified and subscribed to for a channel. + */ +export declare interface Objects { + /** + * Retrieves the root {@link LiveMap} object for Objects on a channel. + * + * A type parameter can be provided to describe the structure of the Objects on the channel. By default, it uses types from the globally defined `AblyObjectsTypes` interface. + * + * You can specify custom types for Objects by defining a global `AblyObjectsTypes` interface with a `root` property that conforms to {@link LiveMapType}. + * + * Example: + * + * ```typescript + * import { LiveCounter } from 'ably'; + * + * type MyRoot = { + * myTypedKey: LiveCounter; + * }; + * + * declare global { + * export interface AblyObjectsTypes { + * root: MyRoot; + * } + * } + * ``` + * + * @returns A promise which, upon success, will be fulfilled with a {@link LiveMap} object. Upon failure, the promise will be rejected with an {@link ErrorInfo} object which explains the error. + * @experimental + */ + getRoot(): Promise>; + + /** + * Creates a new {@link LiveMap} object instance with the provided entries. + * + * @param entries - The initial entries for the new {@link LiveMap} object. + * @returns A promise which, upon success, will be fulfilled with a {@link LiveMap} object. Upon failure, the promise will be rejected with an {@link ErrorInfo} object which explains the error. + * @experimental + */ + createMap(entries?: T): Promise>; + + /** + * Creates a new {@link LiveCounter} object instance with the provided `count` value. + * + * @param count - The initial value for the new {@link LiveCounter} object. + * @returns A promise which, upon success, will be fulfilled with a {@link LiveCounter} object. Upon failure, the promise will be rejected with an {@link ErrorInfo} object which explains the error. + * @experimental + */ + createCounter(count?: number): Promise; + + /** + * Allows you to group multiple operations together and send them to the Ably service in a single channel message. + * As a result, other clients will receive the changes as a single channel message after the batch function has completed. + * + * This method accepts a synchronous callback, which is provided with a {@link BatchContext} object. + * Use the context object to access Objects on a channel and batch operations for them. + * + * The objects' data is not modified inside the callback function. Instead, the objects will be updated + * when the batched operations are applied by the Ably service and echoed back to the client. + * + * @param callback - A batch callback function used to group operations together. Cannot be an `async` function. + * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + * @experimental + */ + batch(callback: BatchCallback): Promise; + + /** + * Registers the provided listener for the specified event. If `on()` is called more than once with the same listener and event, the listener is added multiple times to its listener registry. Therefore, as an example, assuming the same listener is registered twice using `on()`, and an event is emitted once, the listener would be invoked twice. + * + * @param event - The named event to listen for. + * @param callback - The event listener. + * @returns A {@link OnObjectsEventResponse} object that allows the provided listener to be deregistered from future updates. + * @experimental + */ + on(event: ObjectsEvent, callback: ObjectsEventCallback): OnObjectsEventResponse; + + /** + * Removes all registrations that match both the specified listener and the specified event. + * + * @param event - The named event. + * @param callback - The event listener. + * @experimental + */ + off(event: ObjectsEvent, callback: ObjectsEventCallback): void; + + /** + * Deregisters all registrations, for all events and listeners. + * + * @experimental + */ + offAll(): void; +} + +declare global { + /** + * A globally defined interface that allows users to define custom types for Objects. + */ + export interface AblyObjectsTypes { + [key: string]: unknown; + } +} + +/** + * Represents the type of data stored in a {@link LiveMap}. + * It maps string keys to primitive values ({@link PrimitiveObjectValue}), or other {@link LiveObject | LiveObjects}. + */ +export type LiveMapType = { [key: string]: PrimitiveObjectValue | LiveMap | LiveCounter | undefined }; + +/** + * The default type for the `root` object for Objects on a channel, based on the globally defined {@link AblyObjectsTypes} interface. + * + * - If no custom types are provided in `AblyObjectsTypes`, defaults to an untyped root map representation using the {@link LiveMapType} interface. + * - If a `root` type exists in `AblyObjectsTypes` and conforms to the {@link LiveMapType} interface, it is used as the type for the `root` object. + * - If the provided `root` type does not match {@link LiveMapType}, a type error message is returned. + */ +export type DefaultRoot = + // we need a way to know when no types were provided by the user. + // we expect a "root" property to be set on AblyObjectsTypes interface, e.g. it won't be "unknown" anymore + unknown extends AblyObjectsTypes['root'] + ? LiveMapType // no custom types provided; use the default untyped map representation for the root + : AblyObjectsTypes['root'] extends LiveMapType + ? AblyObjectsTypes['root'] // "root" property exists, and it is of an expected type, we can use this interface for the root object in Objects. + : `Provided type definition for the "root" object in AblyObjectsTypes is not of an expected LiveMapType`; + +/** + * Object returned from an `on` call, allowing the listener provided in that call to be deregistered. + */ +export declare interface OnObjectsEventResponse { + /** + * Deregisters the listener passed to the `on` call. + * + * @experimental + */ + off(): void; +} + +/** + * Enables grouping multiple Objects operations together by providing `BatchContext*` wrapper objects. + */ +export declare interface BatchContext { + /** + * Mirrors the {@link Objects.getRoot} method and returns a {@link BatchContextLiveMap} wrapper for the root object on a channel. + * + * @returns A {@link BatchContextLiveMap} object. + * @experimental + */ + getRoot(): BatchContextLiveMap; +} + +/** + * A wrapper around the {@link LiveMap} object that enables batching operations inside a {@link BatchCallback}. + */ +export declare interface BatchContextLiveMap { + /** + * Mirrors the {@link LiveMap.get} method and returns the value associated with a key in the map. + * + * @param key - The key to retrieve the value for. + * @returns A {@link LiveObject}, a primitive type (string, number, boolean, or binary data) or `undefined` if the key doesn't exist in a map or the associated {@link LiveObject} has been deleted. Always `undefined` if this map object is deleted. + * @experimental + */ + get(key: TKey): T[TKey] | undefined; + + /** + * Returns the number of key-value pairs in the map. + * + * @experimental + */ + size(): number; + + /** + * Similar to the {@link LiveMap.set} method, but instead, it adds an operation to set a key in the map with the provided value to the current batch, to be sent in a single message to the Ably service. + * + * This does not modify the underlying data of this object. Instead, the change is applied when + * the published operation is echoed back to the client and applied to the object. + * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. + * + * @param key - The key to set the value for. + * @param value - The value to assign to the key. + * @experimental + */ + set(key: TKey, value: T[TKey]): void; + + /** + * Similar to the {@link LiveMap.remove} method, but instead, it adds an operation to remove a key from the map to the current batch, to be sent in a single message to the Ably service. + * + * This does not modify the underlying data of this object. Instead, the change is applied when + * the published operation is echoed back to the client and applied to the object. + * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. + * + * @param key - The key to set the value for. + * @experimental + */ + remove(key: TKey): void; +} + +/** + * A wrapper around the {@link LiveCounter} object that enables batching operations inside a {@link BatchCallback}. + */ +export declare interface BatchContextLiveCounter { + /** + * Returns the current value of the counter. + * + * @experimental + */ + value(): number; + + /** + * Similar to the {@link LiveCounter.increment} method, but instead, it adds an operation to increment the counter value to the current batch, to be sent in a single message to the Ably service. + * + * This does not modify the underlying data of this object. Instead, the change is applied when + * the published operation is echoed back to the client and applied to the object. + * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. + * + * @param amount - The amount by which to increase the counter value. + * @experimental + */ + increment(amount: number): void; + + /** + * An alias for calling {@link BatchContextLiveCounter.increment | BatchContextLiveCounter.increment(-amount)} + * + * @param amount - The amount by which to decrease the counter value. + * @experimental + */ + decrement(amount: number): void; +} + +/** + * The `LiveMap` class represents a key-value map data structure, similar to a JavaScript Map, where all changes are synchronized across clients in realtime. + * Conflicts in a LiveMap are automatically resolved with last-write-wins (LWW) semantics, + * meaning that if two clients update the same key in the map, the update with the most recent timestamp wins. + * + * Keys must be strings. Values can be another {@link LiveObject}, or a primitive type, such as a string, number, boolean, or binary data (see {@link PrimitiveObjectValue}). + */ +export declare interface LiveMap extends LiveObject> { + /** + * Returns the value associated with a given key. Returns `undefined` if the key doesn't exist in a map or if the associated {@link LiveObject} has been deleted. + * + * Always returns undefined if this map object is deleted. + * + * @param key - The key to retrieve the value for. + * @returns A {@link LiveObject}, a primitive type (string, number, boolean, or binary data) or `undefined` if the key doesn't exist in a map or the associated {@link LiveObject} has been deleted. Always `undefined` if this map object is deleted. + * @experimental + */ + get(key: TKey): T[TKey] | undefined; + + /** + * Returns the number of key-value pairs in the map. + * + * @experimental + */ + size(): number; + + /** + * Returns an iterable of key-value pairs for every entry in the map. + * + * @experimental + */ + entries(): IterableIterator<[TKey, T[TKey]]>; + + /** + * Returns an iterable of keys in the map. + * + * @experimental + */ + keys(): IterableIterator; + + /** + * Returns an iterable of values in the map. + * + * @experimental + */ + values(): IterableIterator; + + /** + * Sends an operation to the Ably system to set a key on this `LiveMap` object to a specified value. + * + * This does not modify the underlying data of this object. Instead, the change is applied when + * the published operation is echoed back to the client and applied to the object. + * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. + * + * @param key - The key to set the value for. + * @param value - The value to assign to the key. + * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + * @experimental + */ + set(key: TKey, value: T[TKey]): Promise; + + /** + * Sends an operation to the Ably system to remove a key from this `LiveMap` object. + * + * This does not modify the underlying data of this object. Instead, the change is applied when + * the published operation is echoed back to the client and applied to the object. + * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. + * + * @param key - The key to remove. + * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + * @experimental + */ + remove(key: TKey): Promise; +} + +/** + * Represents an update to a {@link LiveMap} object, describing the keys that were updated or removed. + */ +export declare interface LiveMapUpdate extends LiveObjectUpdate { + /** + * An object containing keys from a `LiveMap` that have changed, along with their change status: + * - `updated` - the value of a key in the map was updated. + * - `removed` - the key was removed from the map. + */ + update: { [keyName in keyof T & string]?: 'updated' | 'removed' }; +} + +/** + * Represents a primitive value that can be stored in a {@link LiveMap}. + * + * For binary data, the resulting type depends on the platform (`Buffer` in Node.js, `ArrayBuffer` elsewhere). + */ +export type PrimitiveObjectValue = string | number | boolean | Buffer | ArrayBuffer; + +/** + * The `LiveCounter` class represents a counter that can be incremented or decremented and is synchronized across clients in realtime. + */ +export declare interface LiveCounter extends LiveObject { + /** + * Returns the current value of the counter. + * + * @experimental + */ + value(): number; + + /** + * Sends an operation to the Ably system to increment the value of this `LiveCounter` object. + * + * This does not modify the underlying data of this object. Instead, the change is applied when + * the published operation is echoed back to the client and applied to the object. + * To get notified when object gets updated, use the {@link LiveObject.subscribe} method. + * + * @param amount - The amount by which to increase the counter value. + * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + * @experimental + */ + increment(amount: number): Promise; + + /** + * An alias for calling {@link LiveCounter.increment | LiveCounter.increment(-amount)} + * + * @param amount - The amount by which to decrease the counter value. + * @returns A promise which resolves upon success of the operation and rejects with an {@link ErrorInfo} object upon its failure. + * @experimental + */ + decrement(amount: number): Promise; +} + +/** + * Represents an update to a {@link LiveCounter} object. + */ +export declare interface LiveCounterUpdate extends LiveObjectUpdate { + /** + * Holds the numerical change to the counter value. + */ + update: { + /** + * The value by which the counter was incremented or decremented. + */ + amount: number; + }; +} + +/** + * Describes the common interface for all conflict-free data structures supported by the Objects. + */ +export declare interface LiveObject { + /** + * Registers a listener that is called each time this LiveObject is updated. + * + * @param listener - An event listener function that is called with an update object whenever this LiveObject is updated. + * @returns A {@link SubscribeResponse} object that allows the provided listener to be deregistered from future updates. + * @experimental + */ + subscribe(listener: LiveObjectUpdateCallback): SubscribeResponse; + + /** + * Deregisters the given listener from updates for this LiveObject. + * + * @param listener - An event listener function. + * @experimental + */ + unsubscribe(listener: LiveObjectUpdateCallback): void; + + /** + * Deregisters all listeners from updates for this LiveObject. + * + * @experimental + */ + unsubscribeAll(): void; + + /** + * Registers the provided listener for the specified event. If `on()` is called more than once with the same listener and event, the listener is added multiple times to its listener registry. Therefore, as an example, assuming the same listener is registered twice using `on()`, and an event is emitted once, the listener would be invoked twice. + * + * @param event - The named event to listen for. + * @param callback - The event listener. + * @returns A {@link OnLiveObjectLifecycleEventResponse} object that allows the provided listener to be deregistered from future updates. + * @experimental + */ + on(event: LiveObjectLifecycleEvent, callback: LiveObjectLifecycleEventCallback): OnLiveObjectLifecycleEventResponse; + + /** + * Removes all registrations that match both the specified listener and the specified event. + * + * @param event - The named event. + * @param callback - The event listener. + * @experimental + */ + off(event: LiveObjectLifecycleEvent, callback: LiveObjectLifecycleEventCallback): void; + + /** + * Deregisters all registrations, for all events and listeners. + * + * @experimental + */ + offAll(): void; +} + +/** + * Represents a generic update object describing the changes that occurred on a LiveObject. + */ +export declare interface LiveObjectUpdate { + /** + * Holds an update object which describe changes applied to the object. + */ + update: any; +} + +/** + * Object returned from a `subscribe` call, allowing the listener provided in that call to be deregistered. + */ +export declare interface SubscribeResponse { + /** + * Deregisters the listener passed to the `subscribe` call. + * + * @experimental + */ + unsubscribe(): void; +} + +/** + * Object returned from an `on` call, allowing the listener provided in that call to be deregistered. + */ +export declare interface OnLiveObjectLifecycleEventResponse { + /** + * Deregisters the listener passed to the `on` call. + * + * @experimental + */ + off(): void; +} + /** * Enables messages to be published and historic messages to be retrieved for a channel. */ @@ -2362,9 +2906,13 @@ export declare interface RealtimeChannel extends EventEmitter= 0.8" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/destroy": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", @@ -4778,7 +4788,8 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/fast-glob": { "version": "3.3.2", @@ -13578,6 +13589,11 @@ "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "dev": true }, + "dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==" + }, "destroy": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", diff --git a/package.json b/package.json index b28cb26050..40815677e9 100644 --- a/package.json +++ b/package.json @@ -29,19 +29,25 @@ "./push": { "types": "./push.d.ts", "import": "./build/push.js" + }, + "./objects": { + "types": "./objects.d.ts", + "import": "./build/objects.js" } }, "files": [ "build/**", "ably.d.ts", - "push.d.ts", + "objects.d.ts", "modular.d.ts", + "push.d.ts", "resources/**", "src/**", "react/**" ], "dependencies": { "@ably/msgpack-js": "^0.4.0", + "dequal": "^2.0.3", "fastestsmallesttextencoderdecoder": "^1.0.22", "got": "^11.8.5", "ulid": "^2.3.0", @@ -138,8 +144,8 @@ "start:react": "npx vite serve", "grunt": "grunt", "test": "npm run test:node", - "test:node": "npm run build:node && npm run build:push && mocha", - "test:grep": "npm run build:node && npm run build:push && mocha --grep", + "test:node": "npm run build:node && npm run build:push && npm run build:objects && mocha", + "test:grep": "npm run build:node && npm run build:push && npm run build:objects && mocha --grep", "test:node:skip-build": "mocha", "test:webserver": "grunt test:webserver", "test:playwright": "node test/support/runPlaywrightTests.js", @@ -153,6 +159,7 @@ "build:react:mjs": "tsc --project src/platform/react-hooks/tsconfig.mjs.json && cp src/platform/react-hooks/res/package.mjs.json react/mjs/package.json", "build:react:cjs": "tsc --project src/platform/react-hooks/tsconfig.cjs.json && cp src/platform/react-hooks/res/package.cjs.json react/cjs/package.json", "build:push": "grunt build:push", + "build:objects": "grunt build:objects", "requirejs": "grunt requirejs", "lint": "eslint .", "lint:fix": "eslint --fix .", diff --git a/scripts/cdn_deploy.js b/scripts/cdn_deploy.js index 761f2e379b..fb32e764a7 100755 --- a/scripts/cdn_deploy.js +++ b/scripts/cdn_deploy.js @@ -21,7 +21,7 @@ async function run() { // Comma separated directories (relative to `path`) to exclude from upload excludeDirs: 'node_modules,.git', // Regex to match files against for upload - fileRegex: '^(ably|push\\.umd)?(\\.min)?\\.js$', + fileRegex: '^(ably|push\\.umd|objects\\.umd)?(\\.min)?\\.js$', ...argv, }; diff --git a/scripts/moduleReport.ts b/scripts/moduleReport.ts index 9caaba46a6..ef7e6ec8bb 100644 --- a/scripts/moduleReport.ts +++ b/scripts/moduleReport.ts @@ -6,7 +6,7 @@ import { gzip } from 'zlib'; import Table from 'cli-table'; // The maximum size we allow for a minimal useful Realtime bundle (i.e. one that can subscribe to a channel) -const minimalUsefulRealtimeBundleSizeThresholdsKiB = { raw: 100, gzip: 31 }; +const minimalUsefulRealtimeBundleSizeThresholdsKiB = { raw: 102, gzip: 31 }; const baseClientNames = ['BaseRest', 'BaseRealtime']; @@ -39,6 +39,18 @@ const functions = [ { name: 'constructPresenceMessage', transitiveImports: [] }, ]; +// List of all buildable plugins available as a separate export +interface PluginInfo { + description: string; + path: string; + external?: string[]; +} + +const buildablePlugins: Record<'push' | 'objects', PluginInfo> = { + push: { description: 'Push', path: './build/push.js', external: ['ulid'] }, + objects: { description: 'Objects', path: './build/objects.js', external: ['dequal'] }, +}; + function formatBytes(bytes: number) { const kibibytes = bytes / 1024; const formatted = kibibytes.toFixed(2); @@ -72,7 +84,7 @@ function getModularBundleInfo(exports: string[]): BundleInfo { } // Uses esbuild to create a bundle containing the named exports from a given module -function getBundleInfo(modulePath: string, exports?: string[]): BundleInfo { +function getBundleInfo(modulePath: string, exports?: string[], external?: string[]): BundleInfo { const outfile = exports ? exports.join('') : 'all'; const exportTarget = exports ? `{ ${exports.join(', ')} }` : '*'; const result = esbuild.buildSync({ @@ -86,7 +98,7 @@ function getBundleInfo(modulePath: string, exports?: string[]): BundleInfo { outfile, write: false, sourcemap: 'external', - external: ['ulid'], + external, }); const pathHasBase = (component: string) => { @@ -185,22 +197,30 @@ async function calculateAndCheckFunctionSizes(): Promise { return output; } -async function calculatePushPluginSize(): Promise { +async function calculatePluginSize(options: PluginInfo): Promise { const output: Output = { tableRows: [], errors: [] }; - const pushPluginBundleInfo = getBundleInfo('./build/push.js'); + const pluginBundleInfo = getBundleInfo(options.path, undefined, options.external); const sizes = { - rawByteSize: pushPluginBundleInfo.byteSize, - gzipEncodedByteSize: (await promisify(gzip)(pushPluginBundleInfo.code)).byteLength, + rawByteSize: pluginBundleInfo.byteSize, + gzipEncodedByteSize: (await promisify(gzip)(pluginBundleInfo.code)).byteLength, }; output.tableRows.push({ - description: 'Push', + description: options.description, sizes: sizes, }); return output; } +async function calculatePushPluginSize(): Promise { + return calculatePluginSize(buildablePlugins.push); +} + +async function calculateObjectsPluginSize(): Promise { + return calculatePluginSize(buildablePlugins.objects); +} + async function calculateAndCheckMinimalUsefulRealtimeBundleSize(): Promise { const output: Output = { tableRows: [], errors: [] }; @@ -287,7 +307,8 @@ async function checkBaseRealtimeFiles() { } async function checkPushPluginFiles() { - const pushPluginBundleInfo = getBundleInfo('./build/push.js'); + const { path, external } = buildablePlugins.push; + const pushPluginBundleInfo = getBundleInfo(path, undefined, external); // These are the files that are allowed to contribute >= `threshold` bytes to the Push bundle. const allowedFiles = new Set([ @@ -300,6 +321,29 @@ async function checkPushPluginFiles() { return checkBundleFiles(pushPluginBundleInfo, allowedFiles, 100); } +async function checkObjectsPluginFiles() { + const { path, external } = buildablePlugins.objects; + const pluginBundleInfo = getBundleInfo(path, undefined, external); + + // These are the files that are allowed to contribute >= `threshold` bytes to the Objects bundle. + const allowedFiles = new Set([ + 'src/plugins/objects/batchcontext.ts', + 'src/plugins/objects/batchcontextlivecounter.ts', + 'src/plugins/objects/batchcontextlivemap.ts', + 'src/plugins/objects/index.ts', + 'src/plugins/objects/livecounter.ts', + 'src/plugins/objects/livemap.ts', + 'src/plugins/objects/liveobject.ts', + 'src/plugins/objects/objectid.ts', + 'src/plugins/objects/objectmessage.ts', + 'src/plugins/objects/objects.ts', + 'src/plugins/objects/objectspool.ts', + 'src/plugins/objects/syncobjectsdatapool.ts', + ]); + + return checkBundleFiles(pluginBundleInfo, allowedFiles, 100); +} + async function checkBundleFiles(bundleInfo: BundleInfo, allowedFiles: Set, thresholdBytes: number) { const exploreResult = await runSourceMapExplorer(bundleInfo); @@ -351,6 +395,7 @@ async function checkBundleFiles(bundleInfo: BundleInfo, allowedFiles: Set ({ tableRows: [...accum.tableRows, ...current.tableRows], @@ -359,6 +404,7 @@ async function checkBundleFiles(bundleInfo: BundleInfo, allowedFiles: Set { - if (!this.isTimeOffsetSet() && (queryTime || this.authOptions.queryTime)) { - return this.client.time(); - } else { - return this.getTimestampUsingOffset(); - } - } - - getTimestampUsingOffset() { - return Date.now() + (this.client.serverTimeOffset || 0); - } - - isTimeOffsetSet() { - return this.client.serverTimeOffset !== null; - } - _saveBasicOptions(authOptions: AuthOptions) { this.method = 'basic'; this.key = authOptions.key; @@ -913,7 +891,7 @@ class Auth { /* RSA4b1 -- if we have a server time offset set already, we can * automatically remove expired tokens. Else just use the cached token. If it is * expired Ably will tell us and we'll discard it then. */ - if (!this.isTimeOffsetSet() || !token.expires || token.expires >= this.getTimestampUsingOffset()) { + if (!this.client.isTimeOffsetSet() || !token.expires || token.expires >= this.client.getTimestampUsingOffset()) { Logger.logAction( this.logger, Logger.LOG_MINOR, @@ -1020,6 +998,13 @@ class Auth { ): Promise { return this.client.rest.revokeTokens(specifiers, options); } + + /** + * Same as {@link BaseClient.getTimestamp} but also takes into account {@link Auth.authOptions} + */ + private async _getTimestamp(queryTime: boolean): Promise { + return this.client.getTimestamp(queryTime || !!this.authOptions.queryTime); + } } export default Auth; diff --git a/src/common/lib/client/baseclient.ts b/src/common/lib/client/baseclient.ts index f7d23edf06..41931a5197 100644 --- a/src/common/lib/client/baseclient.ts +++ b/src/common/lib/client/baseclient.ts @@ -18,6 +18,8 @@ import { MsgPack } from 'common/types/msgpack'; import { HTTPRequestImplementations } from 'platform/web/lib/http/http'; import { FilteredSubscriptions } from './filteredsubscriptions'; import type { LocalDevice } from 'plugins/push/pushactivation'; +import EventEmitter from '../util/eventemitter'; +import { MessageEncoding } from '../types/basemessage'; type BatchResult = API.BatchResult; type BatchPublishSpec = API.BatchPublishSpec; @@ -172,6 +174,28 @@ class BaseClient { this.logger.setLog(logOptions.level, logOptions.handler); } + /** + * Get the current time based on the local clock, + * or if the option queryTime is true, return the server time. + * The server time offset from the local time is stored so that + * only one request to the server to get the time is ever needed + */ + async getTimestamp(queryTime: boolean): Promise { + if (!this.isTimeOffsetSet() && queryTime) { + return this.time(); + } + + return this.getTimestampUsingOffset(); + } + + getTimestampUsingOffset(): number { + return Date.now() + (this.serverTimeOffset || 0); + } + + isTimeOffsetSet(): boolean { + return this.serverTimeOffset !== null; + } + static Platform = Platform; /** @@ -182,6 +206,8 @@ class BaseClient { Logger = Logger; Defaults = Defaults; Utils = Utils; + EventEmitter = EventEmitter; + MessageEncoding = MessageEncoding; } export default BaseClient; diff --git a/src/common/lib/client/baserealtime.ts b/src/common/lib/client/baserealtime.ts index 8afbd429df..af3bdc78b1 100644 --- a/src/common/lib/client/baserealtime.ts +++ b/src/common/lib/client/baserealtime.ts @@ -13,12 +13,14 @@ import { ModularPlugins, RealtimePresencePlugin } from './modularplugins'; import { TransportNames } from 'common/constants/TransportName'; import { TransportImplementations } from 'common/platform'; import Defaults from '../util/defaults'; +import type * as ObjectsPlugin from 'plugins/objects'; /** `BaseRealtime` is an export of the tree-shakable version of the SDK, and acts as the base class for the `DefaultRealtime` class exported by the non tree-shakable version. */ class BaseRealtime extends BaseClient { readonly _RealtimePresence: RealtimePresencePlugin | null; + readonly _objectsPlugin: typeof ObjectsPlugin | null; // Extra transport implementations available to this client, in addition to those in Platform.Transports.bundledImplementations readonly _additionalTransportImplementations: TransportImplementations; _channels: any; @@ -58,6 +60,7 @@ class BaseRealtime extends BaseClient { this._additionalTransportImplementations = BaseRealtime.transportImplementationsFromPlugins(this.options.plugins); this._RealtimePresence = this.options.plugins?.RealtimePresence ?? null; + this._objectsPlugin = this.options.plugins?.Objects ?? null; this.connection = new Connection(this, this.options); this._channels = new Channels(this); if (this.options.autoConnect !== false) this.connect(); diff --git a/src/common/lib/client/defaultrealtime.ts b/src/common/lib/client/defaultrealtime.ts index f4231cf131..ba860560e5 100644 --- a/src/common/lib/client/defaultrealtime.ts +++ b/src/common/lib/client/defaultrealtime.ts @@ -20,6 +20,7 @@ import Annotation, { WireAnnotation } from '../types/annotation'; import { Http } from 'common/types/http'; import Defaults from '../util/defaults'; import Logger from '../util/logger'; +import { MessageEncoding } from '../types/basemessage'; /** `DefaultRealtime` is the class that the non tree-shakable version of the SDK exports as `Realtime`. It ensures that this version of the SDK includes all of the functionality which is optionally available in the tree-shakable version. @@ -79,4 +80,5 @@ export class DefaultRealtime extends BaseRealtime { // Used by tests static _Http = Http; static _PresenceMap = PresenceMap; + static _MessageEncoding = MessageEncoding; } diff --git a/src/common/lib/client/modularplugins.ts b/src/common/lib/client/modularplugins.ts index 7c10f70cff..99c0a3c5b3 100644 --- a/src/common/lib/client/modularplugins.ts +++ b/src/common/lib/client/modularplugins.ts @@ -10,7 +10,8 @@ import { FilteredSubscriptions } from './filteredsubscriptions'; import PresenceMessage, { WirePresenceMessage } from '../types/presencemessage'; import Annotation, { WireAnnotation } from '../types/annotation'; import { TransportCtor } from '../transport/transport'; -import * as PushPlugin from 'plugins/push'; +import type * as PushPlugin from 'plugins/push'; +import type * as ObjectsPlugin from 'plugins/objects'; export interface PresenceMessagePlugin { PresenceMessage: typeof PresenceMessage; @@ -40,6 +41,7 @@ export interface ModularPlugins { FetchRequest?: typeof fetchRequest; MessageInteractions?: typeof FilteredSubscriptions; Push?: typeof PushPlugin; + Objects?: typeof ObjectsPlugin; } export const allCommonModularPlugins: ModularPlugins = { Rest }; diff --git a/src/common/lib/client/realtimeannotations.ts b/src/common/lib/client/realtimeannotations.ts index a57074bb93..a4f86d30bc 100644 --- a/src/common/lib/client/realtimeannotations.ts +++ b/src/common/lib/client/realtimeannotations.ts @@ -26,7 +26,7 @@ class RealtimeAnnotations { const annotation = constructValidateAnnotation(msgOrSerial, annotationValues); const wireAnnotation = await annotation.encode(); - this.channel._throwIfUnpublishableState(); + this.channel.throwIfUnpublishableState(); Logger.logAction( this.logger, diff --git a/src/common/lib/client/realtimechannel.ts b/src/common/lib/client/realtimechannel.ts index 08d84c4565..8a95b2ce52 100644 --- a/src/common/lib/client/realtimechannel.ts +++ b/src/common/lib/client/realtimechannel.ts @@ -17,6 +17,7 @@ import { normaliseChannelOptions } from '../util/defaults'; import { PaginatedResult } from './paginatedresource'; import type { PushChannel } from 'plugins/push'; import type { WirePresenceMessage } from '../types/presencemessage'; +import type { Objects, ObjectMessage } from 'plugins/objects'; import type RealtimePresence from './realtimepresence'; import type RealtimeAnnotations from './realtimeannotations'; @@ -95,6 +96,7 @@ class RealtimeChannel extends EventEmitter { retryTimer?: number | NodeJS.Timeout | null; retryCount: number = 0; _push?: PushChannel; + _objects?: Objects; constructor(client: BaseRealtime, name: string, options?: API.ChannelOptions) { super(client.logger); @@ -134,6 +136,10 @@ class RealtimeChannel extends EventEmitter { if (client.options.plugins?.Push) { this._push = new client.options.plugins.Push.PushChannel(this); } + + if (client.options.plugins?.Objects) { + this._objects = new client.options.plugins.Objects.Objects(this); + } } get push() { @@ -143,6 +149,13 @@ class RealtimeChannel extends EventEmitter { return this._push; } + get objects() { + if (!this._objects) { + Utils.throwMissingPluginError('Objects'); + } + return this._objects; + } + invalidStateError(): ErrorInfo { return new ErrorInfo( 'Channel operation failed as channel state is ' + this.state, @@ -249,17 +262,13 @@ class RealtimeChannel extends EventEmitter { const size = getMessagesSize(wireMessages); if (size > maxMessageSize) { throw new ErrorInfo( - 'Maximum size of messages that can be published at once exceeded ( was ' + - size + - ' bytes; limit is ' + - maxMessageSize + - ' bytes)', + `Maximum size of messages that can be published at once exceeded (was ${size} bytes; limit is ${maxMessageSize} bytes)`, 40009, 400, ); } - this._throwIfUnpublishableState(); + this.throwIfUnpublishableState(); Logger.logAction( this.logger, @@ -272,7 +281,7 @@ class RealtimeChannel extends EventEmitter { return this.sendMessage(pm); } - _throwIfUnpublishableState(): void { + throwIfUnpublishableState(): void { if (!this.connectionManager.activeState()) { throw this.connectionManager.getError(); } @@ -497,12 +506,22 @@ class RealtimeChannel extends EventEmitter { return this.sendMessage(msg); } + sendState(objectMessages: ObjectMessage[]): Promise { + const msg = protocolMessageFromValues({ + action: actions.OBJECT, + channel: this.name, + state: objectMessages, + }); + return this.sendMessage(msg); + } + // Access to this method is synchronised by ConnectionManager#processChannelMessage, in order to synchronise access to the state stored in _decodingContext. async processMessage(message: ProtocolMessage): Promise { if ( message.action === actions.ATTACHED || message.action === actions.MESSAGE || message.action === actions.PRESENCE || + message.action === actions.OBJECT || message.action === actions.ANNOTATION ) { // RTL15b @@ -521,12 +540,18 @@ class RealtimeChannel extends EventEmitter { const resumed = message.hasFlag('RESUMED'); const hasPresence = message.hasFlag('HAS_PRESENCE'); const hasBacklog = message.hasFlag('HAS_BACKLOG'); + const hasObjects = message.hasFlag('HAS_OBJECTS'); if (this.state === 'attached') { if (!resumed) { - /* On a loss of continuity, the presence set needs to be re-synced */ + // we have lost continuity. + // the presence set needs to be re-synced if (this._presence) { this._presence.onAttached(hasPresence); } + // the Objects tree needs to be re-synced + if (this._objects) { + this._objects.onAttached(hasObjects); + } } const change = new ChannelStateChange(this.state, this.state, resumed, hasBacklog, message.error); this._allChannelChanges.emit('update', change); @@ -537,7 +562,7 @@ class RealtimeChannel extends EventEmitter { /* RTL5i: re-send DETACH and remain in the 'detaching' state */ this.checkPendingState(); } else { - this.notifyState('attached', message.error, resumed, hasPresence, hasBacklog); + this.notifyState('attached', message.error, resumed, hasPresence, hasBacklog, hasObjects); } break; } @@ -587,6 +612,36 @@ class RealtimeChannel extends EventEmitter { } break; } + + // OBJECT and OBJECT_SYNC message processing share most of the logic, so group them together + case actions.OBJECT: + case actions.OBJECT_SYNC: { + if (!this._objects || !message.state) { + return; + } + + populateFieldsFromParent(message); + const objectMessages = message.state; + // need to use the active protocol format instead of just client's useBinaryProtocol option, + // as comet transport does not support msgpack and will default to json without changing useBinaryProtocol. + // message processing is done in the same event loop tick up until this point, + // so we can reliably expect an active protocol to exist and be the one that received the object message. + const format = this.client.connection.connectionManager.getActiveTransportFormat()!; + await Promise.all( + objectMessages.map((om) => + this.client._objectsPlugin!.ObjectMessage.decode(om, this.client, this.logger, Logger, Utils, format), + ), + ); + + if (message.action === actions.OBJECT) { + this._objects.handleObjectMessages(objectMessages); + } else { + this._objects.handleObjectSyncMessages(objectMessages, message.channelSerial); + } + + break; + } + case actions.MESSAGE: { //RTL17 if (this.state !== 'attached') { @@ -725,6 +780,7 @@ class RealtimeChannel extends EventEmitter { resumed?: boolean, hasPresence?: boolean, hasBacklog?: boolean, + hasObjects?: boolean, ): void { Logger.logAction( this.logger, @@ -745,6 +801,9 @@ class RealtimeChannel extends EventEmitter { if (this._presence) { this._presence.actOnChannelState(state, hasPresence, reason); } + if (this._objects) { + this._objects.actOnChannelState(state, hasObjects); + } if (state === 'suspended' && this.connectionManager.state.sendEvents) { this.startRetryTimer(); } else { diff --git a/src/common/lib/client/restchannel.ts b/src/common/lib/client/restchannel.ts index 8cdab42b75..6b2fc4a824 100644 --- a/src/common/lib/client/restchannel.ts +++ b/src/common/lib/client/restchannel.ts @@ -126,11 +126,7 @@ class RestChannel { maxMessageSize = options.maxMessageSize; if (size > maxMessageSize) { throw new ErrorInfo( - 'Maximum size of messages that can be published at once exceeded ( was ' + - size + - ' bytes; limit is ' + - maxMessageSize + - ' bytes)', + `Maximum size of messages that can be published at once exceeded (was ${size} bytes; limit is ${maxMessageSize} bytes)`, 40009, 400, ); diff --git a/src/common/lib/transport/comettransport.ts b/src/common/lib/transport/comettransport.ts index ce75be69f3..a4d3d557ca 100644 --- a/src/common/lib/transport/comettransport.ts +++ b/src/common/lib/transport/comettransport.ts @@ -357,6 +357,7 @@ abstract class CometTransport extends Transport { items[i], this.connectionManager.realtime._RealtimePresence, this.connectionManager.realtime._Annotations, + this.connectionManager.realtime._objectsPlugin, ), ); } catch (e) { diff --git a/src/common/lib/transport/connectionmanager.ts b/src/common/lib/transport/connectionmanager.ts index d7e08b4814..3017dd7c71 100644 --- a/src/common/lib/transport/connectionmanager.ts +++ b/src/common/lib/transport/connectionmanager.ts @@ -949,6 +949,10 @@ class ConnectionManager extends EventEmitter { this.clearSessionRecoverData(); } + getActiveTransportFormat(): Utils.Format | undefined { + return this.activeProtocol?.getTransport().format; + } + /********************* * state management *********************/ @@ -1805,7 +1809,13 @@ class ConnectionManager extends EventEmitter { Logger.LOG_MICRO, 'ConnectionManager.send()', - 'queueing msg; ' + stringifyProtocolMessage(msg, this.realtime._RealtimePresence, this.realtime._Annotations), + 'queueing msg; ' + + stringifyProtocolMessage( + msg, + this.realtime._RealtimePresence, + this.realtime._Annotations, + this.realtime._objectsPlugin, + ), ); } this.queue(msg, callback); diff --git a/src/common/lib/transport/protocol.ts b/src/common/lib/transport/protocol.ts index 98ec9507da..daab68d968 100644 --- a/src/common/lib/transport/protocol.ts +++ b/src/common/lib/transport/protocol.ts @@ -21,7 +21,9 @@ export class PendingMessage { this.merged = false; const action = message.action; this.sendAttempted = false; - this.ackRequired = action == actions.MESSAGE || action == actions.PRESENCE || action == actions.ANNOTATION; + this.ackRequired = + typeof action === 'number' && + [actions.MESSAGE, actions.PRESENCE, actions.ANNOTATION, actions.OBJECT].includes(action); } } @@ -82,6 +84,7 @@ class Protocol extends EventEmitter { pendingMessage.message, this.transport.connectionManager.realtime._RealtimePresence, this.transport.connectionManager.realtime._Annotations, + this.transport.connectionManager.realtime._objectsPlugin, ), ); } diff --git a/src/common/lib/transport/transport.ts b/src/common/lib/transport/transport.ts index 005c84e556..8d4dce3781 100644 --- a/src/common/lib/transport/transport.ts +++ b/src/common/lib/transport/transport.ts @@ -132,6 +132,7 @@ abstract class Transport extends EventEmitter { message, this.connectionManager.realtime._RealtimePresence, this.connectionManager.realtime._Annotations, + this.connectionManager.realtime._objectsPlugin, ) + '; connectionId = ' + this.connectionManager.connectionId, diff --git a/src/common/lib/transport/websockettransport.ts b/src/common/lib/transport/websockettransport.ts index 7145bf4b86..3a2aa1c92a 100644 --- a/src/common/lib/transport/websockettransport.ts +++ b/src/common/lib/transport/websockettransport.ts @@ -141,6 +141,7 @@ class WebSocketTransport extends Transport { this.connectionManager.realtime._MsgPack, this.connectionManager.realtime._RealtimePresence, this.connectionManager.realtime._Annotations, + this.connectionManager.realtime._objectsPlugin, this.format, ), ); diff --git a/src/common/lib/types/basemessage.ts b/src/common/lib/types/basemessage.ts index 3ba549ab22..b3cfc69353 100644 --- a/src/common/lib/types/basemessage.ts +++ b/src/common/lib/types/basemessage.ts @@ -1,9 +1,9 @@ import Platform from 'common/platform'; +import * as API from '../../../../ably'; +import { Bufferlike as BrowserBufferlike } from '../../../platform/web/lib/util/bufferutils'; import Logger from '../util/logger'; -import ErrorInfo from './errorinfo'; import * as Utils from '../util/utils'; -import { Bufferlike as BrowserBufferlike } from '../../../platform/web/lib/util/bufferutils'; -import * as API from '../../../../ably'; +import ErrorInfo from './errorinfo'; import { actions } from './protocolmessagecommon'; import type { IUntypedCryptoStatic } from 'common/types/ICryptoStatic'; @@ -60,35 +60,51 @@ export function normalizeCipherOptions( return options ?? {}; } -async function encrypt(msg: T, options: CipherOptions): Promise { - let data = msg.data, - encoding = msg.encoding, - cipher = options.channelCipher; +async function encrypt(msg: T, cipherOptions: CipherOptions): Promise { + const { data, encoding } = await encryptData(msg.data, msg.encoding, cipherOptions); + msg.data = data; + msg.encoding = encoding; + return msg; +} - encoding = encoding ? encoding + '/' : ''; - if (!Platform.BufferUtils.isBuffer(data)) { - data = Platform.BufferUtils.utf8Encode(String(data)); - encoding = encoding + 'utf-8/'; +export async function encryptData( + data: any, + encoding: string | null | undefined, + cipherOptions: CipherOptions, +): Promise<{ data: any; encoding: string | null | undefined }> { + let cipher = cipherOptions.channelCipher; + let dataToEncrypt = data; + let finalEncoding = encoding ? encoding + '/' : ''; + + if (!Platform.BufferUtils.isBuffer(dataToEncrypt)) { + dataToEncrypt = Platform.BufferUtils.utf8Encode(String(dataToEncrypt)); + finalEncoding = finalEncoding + 'utf-8/'; } - const ciphertext = await cipher.encrypt(data); - msg.data = ciphertext; - msg.encoding = encoding + 'cipher+' + cipher.algorithm; - return msg; + + const ciphertext = await cipher.encrypt(dataToEncrypt); + finalEncoding = finalEncoding + 'cipher+' + cipher.algorithm; + + return { + data: ciphertext, + encoding: finalEncoding, + }; } +/** + * Encodes and encrypts message's payload. Mutates the message object. + * Implements RSL4 and RSL5. + */ export async function encode(msg: T, options: unknown): Promise { - const data = msg.data; - const nativeDataType = - typeof data == 'string' || Platform.BufferUtils.isBuffer(data) || data === null || data === undefined; - - if (!nativeDataType) { - if (Utils.isObject(data) || Array.isArray(data)) { - msg.data = JSON.stringify(data); - msg.encoding = msg.encoding ? msg.encoding + '/json' : 'json'; - } else { - throw new ErrorInfo('Data type is unsupported', 40013, 400); - } - } + // RSL4a, supported types + const isNativeDataType = + typeof msg.data == 'string' || + Platform.BufferUtils.isBuffer(msg.data) || + msg.data === null || + msg.data === undefined; + const { data, encoding } = encodeData(msg.data, msg.encoding, isNativeDataType); + + msg.data = data; + msg.encoding = encoding; if (options != null && (options as CipherOptions).cipher) { return encrypt(msg, options as CipherOptions); @@ -97,21 +113,70 @@ export async function encode(msg: T, options: unknown): P } } +export function encodeData( + data: any, + encoding: string | null | undefined, + isNativeDataType: boolean, +): { data: any; encoding: string | null | undefined } { + if (isNativeDataType) { + // nothing to do with the native data types at this point + return { + data, + encoding, + }; + } + + if (Utils.isObject(data) || Array.isArray(data)) { + // RSL4c3 and RSL4d3, encode objects and arrays as strings + return { + data: JSON.stringify(data), + encoding: encoding ? encoding + '/json' : 'json', + }; + } + + // RSL4a, throw an error for unsupported types + throw new ErrorInfo('Data type is unsupported', 40013, 400); +} + export async function decode( message: T, inputContext: CipherOptions | EncodingDecodingContext | ChannelOptions, ): Promise { + // data can be decoded partially and throw an error on a later decoding step. + // so we need to reassign the data and encoding values we got, and only then throw an error if there is one + const { data, encoding, error } = await decodeData(message.data, message.encoding, inputContext); + message.data = data; + message.encoding = encoding; + + if (error) { + throw error; + } +} + +/** + * Implements RSL6 + */ +export async function decodeData( + data: any, + encoding: string | null | undefined, + inputContext: CipherOptions | EncodingDecodingContext | ChannelOptions, +): Promise<{ + error?: ErrorInfo; + data: any; + encoding: string | null | undefined; +}> { const context = normaliseContext(inputContext); + let lastPayload = data; + let decodedData = data; + let finalEncoding = encoding; + let decodingError: ErrorInfo | undefined; - let lastPayload = message.data; - const encoding = message.encoding; if (encoding) { const xforms = encoding.split('/'); - let lastProcessedEncodingIndex, - encodingsToProcess = xforms.length, - data = message.data; - + let lastProcessedEncodingIndex; + let encodingsToProcess = xforms.length; let xform = ''; + try { while ((lastProcessedEncodingIndex = encodingsToProcess) > 0) { // eslint-disable-next-line security/detect-unsafe-regex @@ -120,16 +185,16 @@ export async function decode( xform = match[1]; switch (xform) { case 'base64': - data = Platform.BufferUtils.base64Decode(String(data)); + decodedData = Platform.BufferUtils.base64Decode(String(decodedData)); if (lastProcessedEncodingIndex == xforms.length) { - lastPayload = data; + lastPayload = decodedData; } continue; case 'utf-8': - data = Platform.BufferUtils.utf8Decode(data); + decodedData = Platform.BufferUtils.utf8Decode(decodedData); continue; case 'json': - data = JSON.parse(data); + decodedData = JSON.parse(decodedData); continue; case 'cipher': if ( @@ -143,7 +208,7 @@ export async function decode( if (xformAlgorithm != cipher.algorithm) { throw new Error('Unable to decrypt message with given cipher; incompatible cipher params'); } - data = await cipher.decrypt(data); + decodedData = await cipher.decrypt(decodedData); continue; } else { throw new Error('Unable to decrypt message; not an encrypted channel'); @@ -167,10 +232,12 @@ export async function decode( // vcdiff expects Uint8Arrays, can't copy with ArrayBuffers. const deltaBaseBuffer = Platform.BufferUtils.toBuffer(deltaBase as Buffer); - data = Platform.BufferUtils.toBuffer(data); + decodedData = Platform.BufferUtils.toBuffer(decodedData); - data = Platform.BufferUtils.arrayBufferViewToBuffer(context.plugins.vcdiff.decode(data, deltaBaseBuffer)); - lastPayload = data; + decodedData = Platform.BufferUtils.arrayBufferViewToBuffer( + context.plugins.vcdiff.decode(decodedData, deltaBaseBuffer), + ); + lastPayload = decodedData; } catch (e) { throw new ErrorInfo('Vcdiff delta decode failed with ' + e, 40018, 400); } @@ -181,41 +248,85 @@ export async function decode( } } catch (e) { const err = e as ErrorInfo; - throw new ErrorInfo( - 'Error processing the ' + xform + ' encoding, decoder returned ‘' + err.message + '’', + decodingError = new ErrorInfo( + `Error processing the ${xform} encoding, decoder returned ‘${err.message}’`, err.code || 40013, 400, ); } finally { - message.encoding = + finalEncoding = (lastProcessedEncodingIndex as number) <= 0 ? null : xforms.slice(0, lastProcessedEncodingIndex).join('/'); - message.data = data; } } + + if (decodingError) { + return { + error: decodingError, + data: decodedData, + encoding: finalEncoding, + }; + } + context.baseEncodedPreviousPayload = lastPayload; + return { + data: decodedData, + encoding: finalEncoding, + }; } export function wireToJSON(this: BaseMessage, ...args: any[]): any { - /* encode data to base64 if present and we're returning real JSON; - * although msgpack calls toJSON(), we know it is a stringify() - * call if it has a non-empty arguments list */ - let encoding = this.encoding; - let data = this.data; - if (data && Platform.BufferUtils.isBuffer(data)) { - if (args.length > 0) { - /* stringify call */ - encoding = encoding ? encoding + '/base64' : 'base64'; - data = Platform.BufferUtils.base64Encode(data); - } else { - /* Called by msgpack. toBuffer returns a datatype understandable by - * that platform's msgpack implementation (Buffer in node, Uint8Array - * in browsers) */ - data = Platform.BufferUtils.toBuffer(data); - } - } + // encode message data for wire transmission. we can infer the format used by client by inspecting with what arguments this method was called. + // if JSON encoding is being used, the JSON.stringify() will be called and this toJSON() method will have a non-empty arguments list. + // MSGPack encoding implementation also calls toJSON(), but with an empty arguments list. + const format = args.length > 0 ? Utils.Format.json : Utils.Format.msgpack; + const { data, encoding } = encodeDataForWire(this.data, this.encoding, format); + return Object.assign({}, this, { encoding, data }); } +/** + * Prepares the payload data to be transmitted over the wire to Ably. + * Encodes the data depending on the selected protocol format. + * + * Implements RSL4c1 and RSL4d1 + */ +export function encodeDataForWire( + data: any, + encoding: string | null | undefined, + format: Utils.Format, +): { data: any; encoding: string | null | undefined } { + if (!data || !Platform.BufferUtils.isBuffer(data)) { + // no encoding required for non-buffer payloads + return { + data, + encoding, + }; + } + + if (format === Utils.Format.msgpack) { + // RSL4c1 + // BufferUtils.toBuffer returns a datatype understandable by that platform's msgpack implementation: + // Buffer in node, Uint8Array in browsers + return { + data: Platform.BufferUtils.toBuffer(data), + encoding, + }; + } + + // RSL4d1, encode binary payload as base64 string + return { + data: Platform.BufferUtils.base64Encode(data), + encoding: encoding ? encoding + '/base64' : 'base64', + }; +} + +export const MessageEncoding = { + encryptData, + encodeData, + encodeDataForWire, + decodeData, +}; + // in-place, generally called on the protocol message before decoding export function populateFieldsFromParent(parent: ProtocolMessage) { const { id, connectionId, timestamp } = parent; @@ -233,6 +344,10 @@ export function populateFieldsFromParent(parent: ProtocolMessage) { case actions.ANNOTATION: msgs = parent.annotations!; break; + case actions.OBJECT: + case actions.OBJECT_SYNC: + msgs = parent.state!; + break; default: throw new ErrorInfo('Unexpected action ' + parent.action, 40000, 400); } @@ -259,7 +374,7 @@ export function strMsg(m: any, cls: string) { result += '; data=' + m.data; } else if (Platform.BufferUtils.isBuffer(m.data)) { result += '; data (buffer)=' + Platform.BufferUtils.base64Encode(m.data); - } else { + } else if (typeof m.data !== 'undefined') { result += '; data (json)=' + JSON.stringify(m.data); } } else if (attr && (attr === 'extras' || attr === 'operation')) { diff --git a/src/common/lib/types/protocolmessage.ts b/src/common/lib/types/protocolmessage.ts index dcf7eeb858..873d7d198f 100644 --- a/src/common/lib/types/protocolmessage.ts +++ b/src/common/lib/types/protocolmessage.ts @@ -11,6 +11,8 @@ import RealtimeAnnotations from '../client/realtimeannotations'; import RestAnnotations from '../client/restannotations'; import { flags, flagNames, channelModes, ActionName } from './protocolmessagecommon'; import type { Properties } from '../util/utils'; +import type * as ObjectsPlugin from 'plugins/objects'; +import { MessageEncoding } from './basemessage'; export const serialize = Utils.encodeBody; @@ -29,16 +31,18 @@ export function deserialize( MsgPack: MsgPack | null, presenceMessagePlugin: PresenceMessagePlugin | null, annotationsPlugin: AnnotationsPlugin | null, + objectsPlugin: typeof ObjectsPlugin | null, format?: Utils.Format, ): ProtocolMessage { const deserialized = Utils.decodeBody>(serialized, MsgPack, format); - return fromDeserialized(deserialized, presenceMessagePlugin, annotationsPlugin); + return fromDeserialized(deserialized, presenceMessagePlugin, annotationsPlugin, objectsPlugin); } export function fromDeserialized( deserialized: Record, presenceMessagePlugin: PresenceMessagePlugin | null, annotationsPlugin: AnnotationsPlugin | null, + objectsPlugin: typeof ObjectsPlugin | null, ): ProtocolMessage { let error: ErrorInfo | undefined; if (deserialized.error) { @@ -64,18 +68,36 @@ export function fromDeserialized( ); } - return Object.assign(new ProtocolMessage(), { ...deserialized, presence, messages, annotations, error }); + let state: ObjectsPlugin.ObjectMessage[] | undefined; + if (objectsPlugin && deserialized.state) { + state = objectsPlugin.ObjectMessage.fromValuesArray( + deserialized.state as ObjectsPlugin.ObjectMessage[], + Utils, + MessageEncoding, + ); + } + + return Object.assign(new ProtocolMessage(), { ...deserialized, presence, messages, annotations, state, error }); } /** - * Used by the tests. + * Used internally by the tests. + * + * ObjectsPlugin code can't be included as part of the core library to prevent size growth, + * so if a test needs to build object messages, then it must provide the plugin upon call. */ -export function fromDeserializedIncludingDependencies(deserialized: Record): ProtocolMessage { - return fromDeserialized( - deserialized, - { PresenceMessage, WirePresenceMessage }, - { Annotation, WireAnnotation, RealtimeAnnotations, RestAnnotations }, - ); +export function makeFromDeserializedWithDependencies(dependencies?: { ObjectsPlugin: typeof ObjectsPlugin | null }) { + return (deserialized: Record): ProtocolMessage => { + return fromDeserialized( + deserialized, + { + PresenceMessage, + WirePresenceMessage, + }, + { Annotation, WireAnnotation, RealtimeAnnotations, RestAnnotations }, + dependencies?.ObjectsPlugin ?? null, + ); + }; } export function fromValues(values: Properties): ProtocolMessage { @@ -86,6 +108,7 @@ export function stringify( msg: any, presenceMessagePlugin: PresenceMessagePlugin | null, annotationsPlugin: AnnotationsPlugin | null, + objectsPlugin: typeof ObjectsPlugin | null, ): string { let result = '[ProtocolMessage'; if (msg.action !== undefined) result += '; action=' + ActionName[msg.action] || msg.action; @@ -103,6 +126,10 @@ export function stringify( if (msg.annotations && annotationsPlugin) { result += '; annotations=' + toStringArray(annotationsPlugin.WireAnnotation.fromValuesArray(msg.annotations)); } + if (msg.state && objectsPlugin) { + result += + '; state=' + toStringArray(objectsPlugin.ObjectMessage.fromValuesArray(msg.state, Utils, MessageEncoding)); + } if (msg.error) result += '; error=' + ErrorInfo.fromValues(msg.error).toString(); if (msg.auth && msg.auth.accessToken) result += '; token=' + msg.auth.accessToken; if (msg.flags) result += '; flags=' + flagNames.filter(msg.hasFlag).join(','); @@ -134,9 +161,15 @@ class ProtocolMessage { channelSerial?: string | null; msgSerial?: number; messages?: WireMessage[]; - // This will be undefined if we skipped decoding this property due to user not requesting presence functionality — see `fromDeserialized` + /** + * This will be undefined if we skipped decoding this property due to user not requesting Presence functionality — see {@link fromDeserialized} + */ presence?: WirePresenceMessage[]; annotations?: WireAnnotation[]; + /** + * This will be undefined if we skipped decoding this property due to user not requesting Objects functionality — see {@link fromDeserialized} + */ + state?: ObjectsPlugin.ObjectMessage[]; auth?: unknown; connectionDetails?: Record; params?: Record; diff --git a/src/common/lib/types/protocolmessagecommon.ts b/src/common/lib/types/protocolmessagecommon.ts index bb1aaabc7c..8c1e43a64d 100644 --- a/src/common/lib/types/protocolmessagecommon.ts +++ b/src/common/lib/types/protocolmessagecommon.ts @@ -21,8 +21,8 @@ export const actions = { SYNC: 16, AUTH: 17, ACTIVATE: 18, - STATE: 19, - STATE_SYNC: 20, + OBJECT: 19, + OBJECT_SYNC: 20, ANNOTATION: 21, }; @@ -38,6 +38,7 @@ export const flags: { [key: string]: number } = { RESUMED: 1 << 2, TRANSIENT: 1 << 4, ATTACH_RESUME: 1 << 5, + HAS_OBJECTS: 1 << 7, /* Channel mode flags */ PRESENCE: 1 << 16, PUBLISH: 1 << 17, @@ -45,6 +46,8 @@ export const flags: { [key: string]: number } = { PRESENCE_SUBSCRIBE: 1 << 19, ANNOTATION_PUBLISH: 1 << 21, ANNOTATION_SUBSCRIBE: 1 << 22, + OBJECT_SUBSCRIBE: 1 << 24, + OBJECT_PUBLISH: 1 << 25, }; export const flagNames = Object.keys(flags); @@ -55,7 +58,9 @@ flags.MODE_ALL = flags.SUBSCRIBE | flags.PRESENCE_SUBSCRIBE | flags.ANNOTATION_PUBLISH | - flags.ANNOTATION_SUBSCRIBE; + flags.ANNOTATION_SUBSCRIBE | + flags.OBJECT_SUBSCRIBE | + flags.OBJECT_PUBLISH; export const channelModes = [ 'PRESENCE', @@ -64,4 +69,6 @@ export const channelModes = [ 'PRESENCE_SUBSCRIBE', 'ANNOTATION_PUBLISH', 'ANNOTATION_SUBSCRIBE', + 'OBJECT_SUBSCRIBE', + 'OBJECT_PUBLISH', ]; diff --git a/src/common/lib/util/utils.ts b/src/common/lib/util/utils.ts index ef2a6ccf10..351daa96b5 100644 --- a/src/common/lib/util/utils.ts +++ b/src/common/lib/util/utils.ts @@ -1,4 +1,4 @@ -import Platform from 'common/platform'; +import Platform, { Bufferlike } from 'common/platform'; import ErrorInfo, { PartialErrorInfo } from 'common/lib/types/errorinfo'; import { ModularPlugins } from '../client/modularplugins'; import { MsgPack } from 'common/types/msgpack'; @@ -165,15 +165,6 @@ export function arrIntersectOb(arr: Array, ob: Partial(arr1: Array, arr2: Array): Array { - const result = []; - for (let i = 0; i < arr1.length; i++) { - const element = arr1[i]; - if (arr2.indexOf(element) == -1) result.push(element); - } - return result; -} - export function arrDeleteValue(arr: Array, val: T): boolean { const idx = arr.indexOf(val); const res = idx != -1; @@ -288,15 +279,31 @@ export function inspectBody(body: unknown): string { } } -/* Data is assumed to be either a string or a buffer. */ -export function dataSizeBytes(data: string | Buffer): number { +/** + * Data is assumed to be either a string, a number, a boolean or a buffer. + * + * Returns the byte size of the provided data based on the spec: + * - TM6a - size of the string is byte length of the string + * - TM6c - size of the buffer is its size in bytes + * - OD3c - size of a number is 8 bytes + * - OD3d - size of a boolean is 1 byte + */ +export function dataSizeBytes(data: string | number | boolean | Bufferlike): number { if (Platform.BufferUtils.isBuffer(data)) { return Platform.BufferUtils.byteLength(data); } if (typeof data === 'string') { return Platform.Config.stringByteSize(data); } - throw new Error('Expected input of Utils.dataSizeBytes to be a buffer or string, but was: ' + typeof data); + if (typeof data === 'number') { + return 8; + } + if (typeof data === 'boolean') { + return 1; + } + throw new Error( + `Expected input of Utils.dataSizeBytes to be a string, a number, a boolean or a buffer, but was: ${typeof data}`, + ); } export function cheapRandStr(): string { diff --git a/src/common/types/IBufferUtils.ts b/src/common/types/IBufferUtils.ts index be573a44f8..8f9b8010bd 100644 --- a/src/common/types/IBufferUtils.ts +++ b/src/common/types/IBufferUtils.ts @@ -8,6 +8,7 @@ export default interface IBufferUtils { toBuffer: (buffer: Bufferlike) => ToBufferOutput; toArrayBuffer: (buffer: Bufferlike) => ArrayBuffer; base64Encode: (buffer: Bufferlike) => string; + base64UrlEncode: (buffer: Bufferlike) => string; base64Decode: (string: string) => Output; hexEncode: (buffer: Bufferlike) => string; hexDecode: (string: string) => Output; @@ -19,5 +20,7 @@ export default interface IBufferUtils { * Returns ArrayBuffer on browser and Buffer on Node.js */ arrayBufferViewToBuffer: (arrayBufferView: ArrayBufferView) => Bufferlike; + concat(buffers: Bufferlike[]): Output; + sha256(message: Bufferlike): Output; hmacSha256(message: Bufferlike, key: Bufferlike): Output; } diff --git a/src/platform/nativescript/index.ts b/src/platform/nativescript/index.ts index 5a57dbe073..448d513d78 100644 --- a/src/platform/nativescript/index.ts +++ b/src/platform/nativescript/index.ts @@ -3,7 +3,7 @@ import { DefaultRest } from '../../common/lib/client/defaultrest'; import { DefaultRealtime } from '../../common/lib/client/defaultrealtime'; import Platform from '../../common/platform'; import ErrorInfo from '../../common/lib/types/errorinfo'; -import { fromDeserializedIncludingDependencies as protocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; +import { makeFromDeserializedWithDependencies as makeProtocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; // Platform Specific import BufferUtils from '../web/lib/util/bufferutils'; @@ -52,5 +52,5 @@ export default { Rest: DefaultRest, Realtime: DefaultRealtime, msgpack, - protocolMessageFromDeserialized, + makeProtocolMessageFromDeserialized, }; diff --git a/src/platform/nodejs/index.ts b/src/platform/nodejs/index.ts index 057d412e66..cca312e1e7 100644 --- a/src/platform/nodejs/index.ts +++ b/src/platform/nodejs/index.ts @@ -3,7 +3,7 @@ import { DefaultRest } from '../../common/lib/client/defaultrest'; import { DefaultRealtime } from '../../common/lib/client/defaultrealtime'; import Platform from '../../common/platform'; import ErrorInfo from '../../common/lib/types/errorinfo'; -import { fromDeserializedIncludingDependencies as protocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; +import { makeFromDeserializedWithDependencies as makeProtocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; // Platform Specific import BufferUtils from './lib/util/bufferutils'; @@ -46,5 +46,5 @@ module.exports = { Rest: DefaultRest, Realtime: DefaultRealtime, msgpack: null, - protocolMessageFromDeserialized, + makeProtocolMessageFromDeserialized, }; diff --git a/src/platform/nodejs/lib/util/bufferutils.ts b/src/platform/nodejs/lib/util/bufferutils.ts index 52b46ba2b1..82ba2f2875 100644 --- a/src/platform/nodejs/lib/util/bufferutils.ts +++ b/src/platform/nodejs/lib/util/bufferutils.ts @@ -17,6 +17,10 @@ class BufferUtils implements IBufferUtils { return this.toBuffer(buffer).toString('base64'); } + base64UrlEncode(buffer: Bufferlike): string { + return this.toBuffer(buffer).toString('base64url'); + } + areBuffersEqual(buffer1: Bufferlike, buffer2: Bufferlike): boolean { if (!buffer1 || !buffer2) return false; return this.toBuffer(buffer1).compare(this.toBuffer(buffer2)) == 0; @@ -70,14 +74,21 @@ class BufferUtils implements IBufferUtils { return Buffer.from(string, 'utf8'); } + concat(buffers: Bufferlike[]): Output { + return Buffer.concat(buffers.map((x) => this.toBuffer(x))); + } + + sha256(message: Bufferlike): Output { + const messageBuffer = this.toBuffer(message); + + return crypto.createHash('SHA256').update(messageBuffer).digest(); + } + hmacSha256(message: Bufferlike, key: Bufferlike): Output { const messageBuffer = this.toBuffer(message); const keyBuffer = this.toBuffer(key); - const hmac = crypto.createHmac('SHA256', keyBuffer); - hmac.update(messageBuffer); - - return hmac.digest(); + return crypto.createHmac('SHA256', keyBuffer).update(messageBuffer).digest(); } } diff --git a/src/platform/react-native/index.ts b/src/platform/react-native/index.ts index 4153714d3a..79914a5b67 100644 --- a/src/platform/react-native/index.ts +++ b/src/platform/react-native/index.ts @@ -3,7 +3,7 @@ import { DefaultRest } from '../../common/lib/client/defaultrest'; import { DefaultRealtime } from '../../common/lib/client/defaultrealtime'; import Platform from '../../common/platform'; import ErrorInfo from '../../common/lib/types/errorinfo'; -import { fromDeserializedIncludingDependencies as protocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; +import { makeFromDeserializedWithDependencies as makeProtocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; // Platform Specific import BufferUtils from '../web/lib/util/bufferutils'; @@ -55,5 +55,5 @@ export default { Rest: DefaultRest, Realtime: DefaultRealtime, msgpack, - protocolMessageFromDeserialized, + makeProtocolMessageFromDeserialized, }; diff --git a/src/platform/web/index.ts b/src/platform/web/index.ts index f262373c8c..f1c0ddd57b 100644 --- a/src/platform/web/index.ts +++ b/src/platform/web/index.ts @@ -3,7 +3,7 @@ import { DefaultRest } from '../../common/lib/client/defaultrest'; import { DefaultRealtime } from '../../common/lib/client/defaultrealtime'; import Platform from '../../common/platform'; import ErrorInfo from '../../common/lib/types/errorinfo'; -import { fromDeserializedIncludingDependencies as protocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; +import { makeFromDeserializedWithDependencies as makeProtocolMessageFromDeserialized } from '../../common/lib/types/protocolmessage'; // Platform Specific import BufferUtils from './lib/util/bufferutils'; @@ -45,11 +45,12 @@ if (Platform.Config.agent) { Platform.Defaults.agent += ' ' + Platform.Config.agent; } -export { DefaultRest as Rest, DefaultRealtime as Realtime, msgpack, protocolMessageFromDeserialized, ErrorInfo }; +export { DefaultRest as Rest, DefaultRealtime as Realtime, msgpack, makeProtocolMessageFromDeserialized, ErrorInfo }; export default { ErrorInfo, Rest: DefaultRest, Realtime: DefaultRealtime, msgpack, + makeProtocolMessageFromDeserialized, }; diff --git a/src/platform/web/lib/util/bufferutils.ts b/src/platform/web/lib/util/bufferutils.ts index 062e663515..cccd538c78 100644 --- a/src/platform/web/lib/util/bufferutils.ts +++ b/src/platform/web/lib/util/bufferutils.ts @@ -1,6 +1,6 @@ import Platform from 'common/platform'; import IBufferUtils from 'common/types/IBufferUtils'; -import { hmac as hmacSha256 } from './hmac-sha256'; +import { hmac as hmacSha256, sha256 } from './hmac-sha256'; /* Most BufferUtils methods that return a binary object return an ArrayBuffer * The exception is toBuffer, which returns a Uint8Array */ @@ -116,6 +116,11 @@ class BufferUtils implements IBufferUtils { return this.uint8ViewToBase64(this.toBuffer(buffer)); } + base64UrlEncode(buffer: Bufferlike): string { + // base64url encoding is based on regular base64 with following changes: https://base64.guru/standards/base64url + return this.base64Encode(buffer).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); + } + base64Decode(str: string): Output { if (ArrayBuffer && Platform.Config.atob) { return this.base64ToArrayBuffer(str); @@ -195,6 +200,26 @@ class BufferUtils implements IBufferUtils { return this.toArrayBuffer(arrayBufferView); } + concat(buffers: Bufferlike[]): Output { + const sumLength = buffers.reduce((acc, v) => acc + v.byteLength, 0); + const result = new Uint8Array(sumLength); + let offset = 0; + + for (const buffer of buffers) { + const uint8Array = this.toBuffer(buffer); + // see TypedArray.set for TypedArray argument https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray/set#typedarray + result.set(uint8Array, offset); + offset += uint8Array.byteLength; + } + + return result.buffer; + } + + sha256(message: Bufferlike): Output { + const hash = sha256(this.toBuffer(message)); + return this.toArrayBuffer(hash); + } + hmacSha256(message: Bufferlike, key: Bufferlike): Output { const hash = hmacSha256(this.toBuffer(key), this.toBuffer(message)); return this.toArrayBuffer(hash); diff --git a/src/platform/web/lib/util/hmac-sha256.ts b/src/platform/web/lib/util/hmac-sha256.ts index dd2ac76711..ac69b6ee1e 100644 --- a/src/platform/web/lib/util/hmac-sha256.ts +++ b/src/platform/web/lib/util/hmac-sha256.ts @@ -102,7 +102,7 @@ function rightRotate(word: number, bits: number) { return (word >>> bits) | (word << (32 - bits)); } -function sha256(data: Uint8Array) { +export function sha256(data: Uint8Array): Uint8Array { // Copy default state var STATE = DEFAULT_STATE.slice(); @@ -185,7 +185,7 @@ function sha256(data: Uint8Array) { ); } -export function hmac(key: Uint8Array, data: Uint8Array) { +export function hmac(key: Uint8Array, data: Uint8Array): Uint8Array { if (key.length > 64) key = sha256(key); if (key.length < 64) { diff --git a/src/plugins/index.d.ts b/src/plugins/index.d.ts index c91240b56f..b1a4fa3ce5 100644 --- a/src/plugins/index.d.ts +++ b/src/plugins/index.d.ts @@ -1,5 +1,7 @@ +import Objects from './objects'; import Push from './push'; export interface StandardPlugins { + Objects?: typeof Objects; Push?: typeof Push; } diff --git a/src/plugins/objects/batchcontext.ts b/src/plugins/objects/batchcontext.ts new file mode 100644 index 0000000000..8e6bc2e76c --- /dev/null +++ b/src/plugins/objects/batchcontext.ts @@ -0,0 +1,107 @@ +import type BaseClient from 'common/lib/client/baseclient'; +import type * as API from '../../../ably'; +import { BatchContextLiveCounter } from './batchcontextlivecounter'; +import { BatchContextLiveMap } from './batchcontextlivemap'; +import { LiveCounter } from './livecounter'; +import { LiveMap } from './livemap'; +import { ObjectMessage } from './objectmessage'; +import { Objects } from './objects'; +import { ROOT_OBJECT_ID } from './objectspool'; + +export class BatchContext { + private _client: BaseClient; + /** Maps object ids to the corresponding batch context object wrappers */ + private _wrappedObjects: Map> = new Map(); + private _queuedMessages: ObjectMessage[] = []; + private _isClosed = false; + + constructor( + private _objects: Objects, + private _root: LiveMap, + ) { + this._client = _objects.getClient(); + this._wrappedObjects.set(this._root.getObjectId(), new BatchContextLiveMap(this, this._objects, this._root)); + } + + getRoot(): BatchContextLiveMap { + this._objects.throwIfInvalidAccessApiConfiguration(); + this.throwIfClosed(); + return this.getWrappedObject(ROOT_OBJECT_ID) as BatchContextLiveMap; + } + + /** + * @internal + */ + getWrappedObject(objectId: string): BatchContextLiveCounter | BatchContextLiveMap | undefined { + if (this._wrappedObjects.has(objectId)) { + return this._wrappedObjects.get(objectId); + } + + const originObject = this._objects.getPool().get(objectId); + if (!originObject) { + return undefined; + } + + let wrappedObject: BatchContextLiveCounter | BatchContextLiveMap; + if (originObject instanceof LiveMap) { + wrappedObject = new BatchContextLiveMap(this, this._objects, originObject); + } else if (originObject instanceof LiveCounter) { + wrappedObject = new BatchContextLiveCounter(this, this._objects, originObject); + } else { + throw new this._client.ErrorInfo( + `Unknown LiveObject instance type: objectId=${originObject.getObjectId()}`, + 50000, + 500, + ); + } + + this._wrappedObjects.set(objectId, wrappedObject); + return wrappedObject; + } + + /** + * @internal + */ + throwIfClosed(): void { + if (this.isClosed()) { + throw new this._client.ErrorInfo('Batch is closed', 40000, 400); + } + } + + /** + * @internal + */ + isClosed(): boolean { + return this._isClosed; + } + + /** + * @internal + */ + close(): void { + this._isClosed = true; + } + + /** + * @internal + */ + queueMessage(msg: ObjectMessage): void { + this._queuedMessages.push(msg); + } + + /** + * @internal + */ + async flush(): Promise { + try { + this.close(); + + if (this._queuedMessages.length > 0) { + await this._objects.publish(this._queuedMessages); + } + } finally { + this._wrappedObjects.clear(); + this._queuedMessages = []; + } + } +} diff --git a/src/plugins/objects/batchcontextlivecounter.ts b/src/plugins/objects/batchcontextlivecounter.ts new file mode 100644 index 0000000000..7f7b6ac1d3 --- /dev/null +++ b/src/plugins/objects/batchcontextlivecounter.ts @@ -0,0 +1,41 @@ +import type BaseClient from 'common/lib/client/baseclient'; +import { BatchContext } from './batchcontext'; +import { LiveCounter } from './livecounter'; +import { Objects } from './objects'; + +export class BatchContextLiveCounter { + private _client: BaseClient; + + constructor( + private _batchContext: BatchContext, + private _objects: Objects, + private _counter: LiveCounter, + ) { + this._client = this._objects.getClient(); + } + + value(): number { + this._objects.throwIfInvalidAccessApiConfiguration(); + this._batchContext.throwIfClosed(); + return this._counter.value(); + } + + increment(amount: number): void { + this._objects.throwIfInvalidWriteApiConfiguration(); + this._batchContext.throwIfClosed(); + const msg = LiveCounter.createCounterIncMessage(this._objects, this._counter.getObjectId(), amount); + this._batchContext.queueMessage(msg); + } + + decrement(amount: number): void { + this._objects.throwIfInvalidWriteApiConfiguration(); + this._batchContext.throwIfClosed(); + // do an explicit type safety check here before negating the amount value, + // so we don't unintentionally change the type sent by a user + if (typeof amount !== 'number') { + throw new this._client.ErrorInfo('Counter value decrement should be a number', 40003, 400); + } + + this.increment(-amount); + } +} diff --git a/src/plugins/objects/batchcontextlivemap.ts b/src/plugins/objects/batchcontextlivemap.ts new file mode 100644 index 0000000000..306b313426 --- /dev/null +++ b/src/plugins/objects/batchcontextlivemap.ts @@ -0,0 +1,62 @@ +import type * as API from '../../../ably'; +import { BatchContext } from './batchcontext'; +import { LiveMap } from './livemap'; +import { LiveObject } from './liveobject'; +import { Objects } from './objects'; + +export class BatchContextLiveMap { + constructor( + private _batchContext: BatchContext, + private _objects: Objects, + private _map: LiveMap, + ) {} + + get(key: TKey): T[TKey] | undefined { + this._objects.throwIfInvalidAccessApiConfiguration(); + this._batchContext.throwIfClosed(); + const value = this._map.get(key); + if (value instanceof LiveObject) { + return this._batchContext.getWrappedObject(value.getObjectId()) as T[TKey]; + } else { + return value; + } + } + + size(): number { + this._objects.throwIfInvalidAccessApiConfiguration(); + this._batchContext.throwIfClosed(); + return this._map.size(); + } + + *entries(): IterableIterator<[TKey, T[TKey]]> { + this._objects.throwIfInvalidAccessApiConfiguration(); + this._batchContext.throwIfClosed(); + yield* this._map.entries(); + } + + *keys(): IterableIterator { + this._objects.throwIfInvalidAccessApiConfiguration(); + this._batchContext.throwIfClosed(); + yield* this._map.keys(); + } + + *values(): IterableIterator { + this._objects.throwIfInvalidAccessApiConfiguration(); + this._batchContext.throwIfClosed(); + yield* this._map.values(); + } + + set(key: TKey, value: T[TKey]): void { + this._objects.throwIfInvalidWriteApiConfiguration(); + this._batchContext.throwIfClosed(); + const msg = LiveMap.createMapSetMessage(this._objects, this._map.getObjectId(), key, value); + this._batchContext.queueMessage(msg); + } + + remove(key: TKey): void { + this._objects.throwIfInvalidWriteApiConfiguration(); + this._batchContext.throwIfClosed(); + const msg = LiveMap.createMapRemoveMessage(this._objects, this._map.getObjectId(), key); + this._batchContext.queueMessage(msg); + } +} diff --git a/src/plugins/objects/defaults.ts b/src/plugins/objects/defaults.ts new file mode 100644 index 0000000000..fa3a091014 --- /dev/null +++ b/src/plugins/objects/defaults.ts @@ -0,0 +1,10 @@ +export const DEFAULTS = { + gcInterval: 1000 * 60 * 5, // 5 minutes + /** + * Must be > 2 minutes to ensure we keep tombstones long enough to avoid the possibility of receiving an operation + * with an earlier serial that would not have been applied if the tombstone still existed. + * + * Applies both for map entries tombstones and object tombstones. + */ + gcGracePeriod: 1000 * 60 * 60 * 24, // 24 hours +}; diff --git a/src/plugins/objects/index.ts b/src/plugins/objects/index.ts new file mode 100644 index 0000000000..1b9d27f743 --- /dev/null +++ b/src/plugins/objects/index.ts @@ -0,0 +1,9 @@ +import { ObjectMessage } from './objectmessage'; +import { Objects } from './objects'; + +export { Objects, ObjectMessage }; + +export default { + Objects, + ObjectMessage, +}; diff --git a/src/plugins/objects/livecounter.ts b/src/plugins/objects/livecounter.ts new file mode 100644 index 0000000000..ccc358bc2a --- /dev/null +++ b/src/plugins/objects/livecounter.ts @@ -0,0 +1,337 @@ +import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; +import { ObjectId } from './objectid'; +import { CounterOp, ObjectMessage, ObjectOperation, ObjectOperationAction, ObjectState } from './objectmessage'; +import { Objects } from './objects'; + +export interface LiveCounterData extends LiveObjectData { + data: number; +} + +export interface LiveCounterUpdate extends LiveObjectUpdate { + update: { amount: number }; +} + +export class LiveCounter extends LiveObject { + /** + * Returns a {@link LiveCounter} instance with a 0 value. + * + * @internal + */ + static zeroValue(objects: Objects, objectId: string): LiveCounter { + return new LiveCounter(objects, objectId); + } + + /** + * Returns a {@link LiveCounter} instance based on the provided object state. + * The provided object state must hold a valid counter object data. + * + * @internal + */ + static fromObjectState(objects: Objects, objectState: ObjectState): LiveCounter { + const obj = new LiveCounter(objects, objectState.objectId); + obj.overrideWithObjectState(objectState); + return obj; + } + + /** + * Returns a {@link LiveCounter} instance based on the provided COUNTER_CREATE object operation. + * The provided object operation must hold a valid counter object data. + * + * @internal + */ + static fromObjectOperation(objects: Objects, objectOperation: ObjectOperation): LiveCounter { + const obj = new LiveCounter(objects, objectOperation.objectId); + obj._mergeInitialDataFromCreateOperation(objectOperation); + return obj; + } + + /** + * @internal + */ + static createCounterIncMessage(objects: Objects, objectId: string, amount: number): ObjectMessage { + const client = objects.getClient(); + + if (typeof amount !== 'number' || !Number.isFinite(amount)) { + throw new client.ErrorInfo('Counter value increment should be a valid number', 40003, 400); + } + + const msg = ObjectMessage.fromValues( + { + operation: { + action: ObjectOperationAction.COUNTER_INC, + objectId, + counterOp: { amount }, + } as ObjectOperation, + }, + client.Utils, + client.MessageEncoding, + ); + + return msg; + } + + /** + * @internal + */ + static async createCounterCreateMessage(objects: Objects, count?: number): Promise { + const client = objects.getClient(); + + if (count !== undefined && (typeof count !== 'number' || !Number.isFinite(count))) { + throw new client.ErrorInfo('Counter value should be a valid number', 40003, 400); + } + + const initialValueObj = LiveCounter.createInitialValueObject(count); + const { encodedInitialValue, format } = ObjectMessage.encodeInitialValue(initialValueObj, client); + const nonce = client.Utils.cheapRandStr(); + const msTimestamp = await client.getTimestamp(true); + + const objectId = ObjectId.fromInitialValue( + client.Platform, + 'counter', + encodedInitialValue, + nonce, + msTimestamp, + ).toString(); + + const msg = ObjectMessage.fromValues( + { + operation: { + ...initialValueObj, + action: ObjectOperationAction.COUNTER_CREATE, + objectId, + nonce, + initialValue: encodedInitialValue, + initialValueEncoding: format, + } as ObjectOperation, + }, + client.Utils, + client.MessageEncoding, + ); + + return msg; + } + + /** + * @internal + */ + static createInitialValueObject(count?: number): Pick { + return { + counter: { + count: count ?? 0, + }, + }; + } + + value(): number { + this._objects.throwIfInvalidAccessApiConfiguration(); + return this._dataRef.data; + } + + /** + * Send a COUNTER_INC operation to the realtime system to increment a value on this LiveCounter object. + * + * This does not modify the underlying data of this LiveCounter object. Instead, the change will be applied when + * the published COUNTER_INC operation is echoed back to the client and applied to the object following the regular + * operation application procedure. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. + */ + async increment(amount: number): Promise { + this._objects.throwIfInvalidWriteApiConfiguration(); + const msg = LiveCounter.createCounterIncMessage(this._objects, this.getObjectId(), amount); + return this._objects.publish([msg]); + } + + /** + * An alias for calling {@link LiveCounter.increment | LiveCounter.increment(-amount)} + */ + async decrement(amount: number): Promise { + this._objects.throwIfInvalidWriteApiConfiguration(); + // do an explicit type safety check here before negating the amount value, + // so we don't unintentionally change the type sent by a user + if (typeof amount !== 'number' || !Number.isFinite(amount)) { + throw new this._client.ErrorInfo('Counter value decrement should be a valid number', 40003, 400); + } + + return this.increment(-amount); + } + + /** + * @internal + */ + applyOperation(op: ObjectOperation, msg: ObjectMessage): void { + if (op.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Cannot apply object operation with objectId=${op.objectId}, to this LiveCounter with objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + + const opSerial = msg.serial!; + const opSiteCode = msg.siteCode!; + if (!this._canApplyOperation(opSerial, opSiteCode)) { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveCounter.applyOperation()', + `skipping ${op.action} op: op serial ${opSerial.toString()} <= site serial ${this._siteTimeserials[opSiteCode]?.toString()}; objectId=${this.getObjectId()}`, + ); + return; + } + // should update stored site serial immediately. doesn't matter if we successfully apply the op, + // as it's important to mark that the op was processed by the object + this._siteTimeserials[opSiteCode] = opSerial; + + if (this.isTombstoned()) { + // this object is tombstoned so the operation cannot be applied + return; + } + + let update: LiveCounterUpdate | LiveObjectUpdateNoop; + switch (op.action) { + case ObjectOperationAction.COUNTER_CREATE: + update = this._applyCounterCreate(op); + break; + + case ObjectOperationAction.COUNTER_INC: + if (this._client.Utils.isNil(op.counterOp)) { + this._throwNoPayloadError(op); + // leave an explicit return here, so that TS knows that update object is always set after the switch statement. + return; + } else { + update = this._applyCounterInc(op.counterOp); + } + break; + + case ObjectOperationAction.OBJECT_DELETE: + update = this._applyObjectDelete(); + break; + + default: + throw new this._client.ErrorInfo( + `Invalid ${op.action} op for LiveCounter objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + + this.notifyUpdated(update); + } + + /** + * @internal + */ + overrideWithObjectState(objectState: ObjectState): LiveCounterUpdate | LiveObjectUpdateNoop { + if (objectState.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Invalid object state: object state objectId=${objectState.objectId}; LiveCounter objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + + if (!this._client.Utils.isNil(objectState.createOp)) { + // it is expected that create operation can be missing in the object state, so only validate it when it exists + if (objectState.createOp.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Invalid object state: object state createOp objectId=${objectState.createOp?.objectId}; LiveCounter objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + + if (objectState.createOp.action !== ObjectOperationAction.COUNTER_CREATE) { + throw new this._client.ErrorInfo( + `Invalid object state: object state createOp action=${objectState.createOp?.action}; LiveCounter objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + } + + // object's site serials are still updated even if it is tombstoned, so always use the site serials received from the operation. + // should default to empty map if site serials do not exist on the object state, so that any future operation may be applied to this object. + this._siteTimeserials = objectState.siteTimeserials ?? {}; + + if (this.isTombstoned()) { + // this object is tombstoned. this is a terminal state which can't be overridden. skip the rest of object state message processing + return { noop: true }; + } + + const previousDataRef = this._dataRef; + if (objectState.tombstone) { + // tombstone this object and ignore the data from the object state message + this.tombstone(); + } else { + // override data for this object with data from the object state + this._createOperationIsMerged = false; + this._dataRef = { data: objectState.counter?.count ?? 0 }; + if (!this._client.Utils.isNil(objectState.createOp)) { + this._mergeInitialDataFromCreateOperation(objectState.createOp); + } + } + + // if object got tombstoned, the update object will include all data that got cleared. + // otherwise it is a diff between previous value and new value from object state. + return this._updateFromDataDiff(previousDataRef, this._dataRef); + } + + /** + * @internal + */ + onGCInterval(): void { + // nothing to GC for a counter object + return; + } + + protected _getZeroValueData(): LiveCounterData { + return { data: 0 }; + } + + protected _updateFromDataDiff(prevDataRef: LiveCounterData, newDataRef: LiveCounterData): LiveCounterUpdate { + const counterDiff = newDataRef.data - prevDataRef.data; + return { update: { amount: counterDiff } }; + } + + protected _mergeInitialDataFromCreateOperation(objectOperation: ObjectOperation): LiveCounterUpdate { + // if a counter object is missing for the COUNTER_CREATE op, the initial value is implicitly 0 in this case. + // note that it is intentional to SUM the incoming count from the create op. + // if we got here, it means that current counter instance is missing the initial value in its data reference, + // which we're going to add now. + this._dataRef.data += objectOperation.counter?.count ?? 0; + this._createOperationIsMerged = true; + + return { update: { amount: objectOperation.counter?.count ?? 0 } }; + } + + private _throwNoPayloadError(op: ObjectOperation): void { + throw new this._client.ErrorInfo( + `No payload found for ${op.action} op for LiveCounter objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + + private _applyCounterCreate(op: ObjectOperation): LiveCounterUpdate | LiveObjectUpdateNoop { + if (this._createOperationIsMerged) { + // There can't be two different create operation for the same object id, because the object id + // fully encodes that operation. This means we can safely ignore any new incoming create operations + // if we already merged it once. + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveCounter._applyCounterCreate()', + `skipping applying COUNTER_CREATE op on a counter instance as it was already applied before; objectId=${this.getObjectId()}`, + ); + return { noop: true }; + } + + return this._mergeInitialDataFromCreateOperation(op); + } + + private _applyCounterInc(op: CounterOp): LiveCounterUpdate { + this._dataRef.data += op.amount; + return { update: { amount: op.amount } }; + } +} diff --git a/src/plugins/objects/livemap.ts b/src/plugins/objects/livemap.ts new file mode 100644 index 0000000000..b7525fc3e2 --- /dev/null +++ b/src/plugins/objects/livemap.ts @@ -0,0 +1,907 @@ +import { dequal } from 'dequal'; + +import type { Bufferlike } from 'common/platform'; +import type * as API from '../../../ably'; +import { DEFAULTS } from './defaults'; +import { LiveObject, LiveObjectData, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; +import { ObjectId } from './objectid'; +import { + MapEntry, + MapOp, + MapSemantics, + ObjectMessage, + ObjectOperation, + ObjectOperationAction, + ObjectState, +} from './objectmessage'; +import { Objects } from './objects'; + +export type PrimitiveObjectValue = string | number | boolean | Bufferlike; + +export interface ObjectIdObjectData { + /** A reference to another object, used to support composable object structures. */ + objectId: string; +} + +export interface ValueObjectData { + /** Can be set by the client to indicate that value in `string` or `bytes` field have an encoding. */ + encoding?: string; + /** A primitive boolean leaf value in the object graph. Only one value field can be set. */ + boolean?: boolean; + /** A primitive binary leaf value in the object graph. Only one value field can be set. */ + bytes?: Bufferlike; + /** A primitive number leaf value in the object graph. Only one value field can be set. */ + number?: number; + /** A primitive string leaf value in the object graph. Only one value field can be set. */ + string?: string; +} + +export type ObjectData = ObjectIdObjectData | ValueObjectData; + +export interface LiveMapEntry { + tombstone: boolean; + /** + * Can't use serial from the operation that deleted the entry for the same reason as for {@link LiveObject} tombstones, see explanation there. + */ + tombstonedAt: number | undefined; + timeserial: string | undefined; + data: ObjectData | undefined; +} + +export interface LiveMapData extends LiveObjectData { + data: Map; +} + +export interface LiveMapUpdate extends LiveObjectUpdate { + update: { [keyName in keyof T & string]?: 'updated' | 'removed' }; +} + +export class LiveMap extends LiveObject> { + constructor( + objects: Objects, + private _semantics: MapSemantics, + objectId: string, + ) { + super(objects, objectId); + } + + /** + * Returns a {@link LiveMap} instance with an empty map data. + * + * @internal + */ + static zeroValue(objects: Objects, objectId: string): LiveMap { + return new LiveMap(objects, MapSemantics.LWW, objectId); + } + + /** + * Returns a {@link LiveMap} instance based on the provided object state. + * The provided object state must hold a valid map object data. + * + * @internal + */ + static fromObjectState(objects: Objects, objectState: ObjectState): LiveMap { + const obj = new LiveMap(objects, objectState.map?.semantics!, objectState.objectId); + obj.overrideWithObjectState(objectState); + return obj; + } + + /** + * Returns a {@link LiveMap} instance based on the provided MAP_CREATE object operation. + * The provided object operation must hold a valid map object data. + * + * @internal + */ + static fromObjectOperation( + objects: Objects, + objectOperation: ObjectOperation, + ): LiveMap { + const obj = new LiveMap(objects, objectOperation.map?.semantics!, objectOperation.objectId); + obj._mergeInitialDataFromCreateOperation(objectOperation); + return obj; + } + + /** + * @internal + */ + static createMapSetMessage( + objects: Objects, + objectId: string, + key: TKey, + value: API.LiveMapType[TKey], + ): ObjectMessage { + const client = objects.getClient(); + + LiveMap.validateKeyValue(objects, key, value); + + let objectData: ObjectData; + if (value instanceof LiveObject) { + const typedObjectData: ObjectIdObjectData = { objectId: value.getObjectId() }; + objectData = typedObjectData; + } else { + const typedObjectData: ValueObjectData = {}; + if (typeof value === 'string') { + typedObjectData.string = value; + } else if (typeof value === 'number') { + typedObjectData.number = value; + } else if (typeof value === 'boolean') { + typedObjectData.boolean = value; + } else { + typedObjectData.bytes = value as Bufferlike; + } + objectData = typedObjectData; + } + + const msg = ObjectMessage.fromValues( + { + operation: { + action: ObjectOperationAction.MAP_SET, + objectId, + mapOp: { + key, + data: objectData, + }, + } as ObjectOperation, + }, + client.Utils, + client.MessageEncoding, + ); + + return msg; + } + + /** + * @internal + */ + static createMapRemoveMessage( + objects: Objects, + objectId: string, + key: TKey, + ): ObjectMessage { + const client = objects.getClient(); + + if (typeof key !== 'string') { + throw new client.ErrorInfo('Map key should be string', 40003, 400); + } + + const msg = ObjectMessage.fromValues( + { + operation: { + action: ObjectOperationAction.MAP_REMOVE, + objectId, + mapOp: { key }, + } as ObjectOperation, + }, + client.Utils, + client.MessageEncoding, + ); + + return msg; + } + + /** + * @internal + */ + static validateKeyValue( + objects: Objects, + key: TKey, + value: API.LiveMapType[TKey], + ): void { + const client = objects.getClient(); + + if (typeof key !== 'string') { + throw new client.ErrorInfo('Map key should be string', 40003, 400); + } + + if ( + typeof value !== 'string' && + typeof value !== 'number' && + typeof value !== 'boolean' && + !client.Platform.BufferUtils.isBuffer(value) && + !(value instanceof LiveObject) + ) { + throw new client.ErrorInfo('Map value data type is unsupported', 40013, 400); + } + } + + /** + * @internal + */ + static async createMapCreateMessage(objects: Objects, entries?: API.LiveMapType): Promise { + const client = objects.getClient(); + + if (entries !== undefined && (entries === null || typeof entries !== 'object')) { + throw new client.ErrorInfo('Map entries should be a key-value object', 40003, 400); + } + + Object.entries(entries ?? {}).forEach(([key, value]) => LiveMap.validateKeyValue(objects, key, value)); + + const initialValueObj = LiveMap.createInitialValueObject(entries); + const { encodedInitialValue, format } = ObjectMessage.encodeInitialValue(initialValueObj, client); + const nonce = client.Utils.cheapRandStr(); + const msTimestamp = await client.getTimestamp(true); + + const objectId = ObjectId.fromInitialValue( + client.Platform, + 'map', + encodedInitialValue, + nonce, + msTimestamp, + ).toString(); + + const msg = ObjectMessage.fromValues( + { + operation: { + ...initialValueObj, + action: ObjectOperationAction.MAP_CREATE, + objectId, + nonce, + initialValue: encodedInitialValue, + initialValueEncoding: format, + } as ObjectOperation, + }, + client.Utils, + client.MessageEncoding, + ); + + return msg; + } + + /** + * @internal + */ + static createInitialValueObject(entries?: API.LiveMapType): Pick { + const mapEntries: Record = {}; + + Object.entries(entries ?? {}).forEach(([key, value]) => { + let objectData: ObjectData; + if (value instanceof LiveObject) { + const typedObjectData: ObjectIdObjectData = { objectId: value.getObjectId() }; + objectData = typedObjectData; + } else { + const typedObjectData: ValueObjectData = {}; + if (typeof value === 'string') { + typedObjectData.string = value; + } else if (typeof value === 'number') { + typedObjectData.number = value; + } else if (typeof value === 'boolean') { + typedObjectData.boolean = value; + } else { + typedObjectData.bytes = value as Bufferlike; + } + objectData = typedObjectData; + } + + mapEntries[key] = { + data: objectData, + }; + }); + + return { + map: { + semantics: MapSemantics.LWW, + entries: mapEntries, + }, + }; + } + + /** + * Returns the value associated with the specified key in the underlying Map object. + * + * - If this map object is tombstoned (deleted), `undefined` is returned. + * - If no entry is associated with the specified key, `undefined` is returned. + * - If map entry is tombstoned (deleted), `undefined` is returned. + * - If the value associated with the provided key is an objectId string of another LiveObject, a reference to that LiveObject + * is returned, provided it exists in the local pool and is not tombstoned. Otherwise, `undefined` is returned. + * - If the value is not an objectId, then that value is returned. + */ + // force the key to be of type string as we only allow strings as key in a map + get(key: TKey): T[TKey] | undefined { + this._objects.throwIfInvalidAccessApiConfiguration(); + + if (this.isTombstoned()) { + return undefined as T[TKey]; + } + + const element = this._dataRef.data.get(key); + + if (element === undefined) { + return undefined as T[TKey]; + } + + if (element.tombstone === true) { + return undefined as T[TKey]; + } + + // data always exists for non-tombstoned elements + return this._getResolvedValueFromObjectData(element.data!) as T[TKey]; + } + + size(): number { + this._objects.throwIfInvalidAccessApiConfiguration(); + + let size = 0; + for (const value of this._dataRef.data.values()) { + if (this._isMapEntryTombstoned(value)) { + // should not count tombstoned entries + continue; + } + + size++; + } + + return size; + } + + *entries(): IterableIterator<[TKey, T[TKey]]> { + this._objects.throwIfInvalidAccessApiConfiguration(); + + for (const [key, entry] of this._dataRef.data.entries()) { + if (this._isMapEntryTombstoned(entry)) { + // do not return tombstoned entries + continue; + } + + // data always exists for non-tombstoned elements + const value = this._getResolvedValueFromObjectData(entry.data!) as T[TKey]; + yield [key as TKey, value]; + } + } + + *keys(): IterableIterator { + for (const [key] of this.entries()) { + yield key; + } + } + + *values(): IterableIterator { + for (const [_, value] of this.entries()) { + yield value; + } + } + + /** + * Send a MAP_SET operation to the realtime system to set a key on this LiveMap object to a specified value. + * + * This does not modify the underlying data of this LiveMap object. Instead, the change will be applied when + * the published MAP_SET operation is echoed back to the client and applied to the object following the regular + * operation application procedure. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. + */ + async set(key: TKey, value: T[TKey]): Promise { + this._objects.throwIfInvalidWriteApiConfiguration(); + const msg = LiveMap.createMapSetMessage(this._objects, this.getObjectId(), key, value); + return this._objects.publish([msg]); + } + + /** + * Send a MAP_REMOVE operation to the realtime system to tombstone a key on this LiveMap object. + * + * This does not modify the underlying data of this LiveMap object. Instead, the change will be applied when + * the published MAP_REMOVE operation is echoed back to the client and applied to the object following the regular + * operation application procedure. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. + */ + async remove(key: TKey): Promise { + this._objects.throwIfInvalidWriteApiConfiguration(); + const msg = LiveMap.createMapRemoveMessage(this._objects, this.getObjectId(), key); + return this._objects.publish([msg]); + } + + /** + * @internal + */ + applyOperation(op: ObjectOperation, msg: ObjectMessage): void { + if (op.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Cannot apply object operation with objectId=${op.objectId}, to this LiveMap with objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + + const opSerial = msg.serial!; + const opSiteCode = msg.siteCode!; + if (!this._canApplyOperation(opSerial, opSiteCode)) { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveMap.applyOperation()', + `skipping ${op.action} op: op serial ${opSerial.toString()} <= site serial ${this._siteTimeserials[opSiteCode]?.toString()}; objectId=${this.getObjectId()}`, + ); + return; + } + // should update stored site serial immediately. doesn't matter if we successfully apply the op, + // as it's important to mark that the op was processed by the object + this._siteTimeserials[opSiteCode] = opSerial; + + if (this.isTombstoned()) { + // this object is tombstoned so the operation cannot be applied + return; + } + + let update: LiveMapUpdate | LiveObjectUpdateNoop; + switch (op.action) { + case ObjectOperationAction.MAP_CREATE: + update = this._applyMapCreate(op); + break; + + case ObjectOperationAction.MAP_SET: + if (this._client.Utils.isNil(op.mapOp)) { + this._throwNoPayloadError(op); + // leave an explicit return here, so that TS knows that update object is always set after the switch statement. + return; + } else { + update = this._applyMapSet(op.mapOp, opSerial); + } + break; + + case ObjectOperationAction.MAP_REMOVE: + if (this._client.Utils.isNil(op.mapOp)) { + this._throwNoPayloadError(op); + // leave an explicit return here, so that TS knows that update object is always set after the switch statement. + return; + } else { + update = this._applyMapRemove(op.mapOp, opSerial); + } + break; + + case ObjectOperationAction.OBJECT_DELETE: + update = this._applyObjectDelete(); + break; + + default: + throw new this._client.ErrorInfo( + `Invalid ${op.action} op for LiveMap objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + + this.notifyUpdated(update); + } + + /** + * @internal + */ + overrideWithObjectState(objectState: ObjectState): LiveMapUpdate | LiveObjectUpdateNoop { + if (objectState.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Invalid object state: object state objectId=${objectState.objectId}; LiveMap objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + + if (objectState.map?.semantics !== this._semantics) { + throw new this._client.ErrorInfo( + `Invalid object state: object state map semantics=${objectState.map?.semantics}; LiveMap semantics=${this._semantics}`, + 92000, + 500, + ); + } + + if (!this._client.Utils.isNil(objectState.createOp)) { + // it is expected that create operation can be missing in the object state, so only validate it when it exists + if (objectState.createOp.objectId !== this.getObjectId()) { + throw new this._client.ErrorInfo( + `Invalid object state: object state createOp objectId=${objectState.createOp?.objectId}; LiveMap objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + + if (objectState.createOp.action !== ObjectOperationAction.MAP_CREATE) { + throw new this._client.ErrorInfo( + `Invalid object state: object state createOp action=${objectState.createOp?.action}; LiveMap objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + + if (objectState.createOp.map?.semantics !== this._semantics) { + throw new this._client.ErrorInfo( + `Invalid object state: object state createOp map semantics=${objectState.createOp.map?.semantics}; LiveMap semantics=${this._semantics}`, + 92000, + 500, + ); + } + } + + // object's site serials are still updated even if it is tombstoned, so always use the site serials received from the op. + // should default to empty map if site serials do not exist on the object state, so that any future operation may be applied to this object. + this._siteTimeserials = objectState.siteTimeserials ?? {}; + + if (this.isTombstoned()) { + // this object is tombstoned. this is a terminal state which can't be overridden. skip the rest of object state message processing + return { noop: true }; + } + + const previousDataRef = this._dataRef; + if (objectState.tombstone) { + // tombstone this object and ignore the data from the object state message + this.tombstone(); + } else { + // override data for this object with data from the object state + this._createOperationIsMerged = false; + this._dataRef = this._liveMapDataFromMapEntries(objectState.map?.entries ?? {}); + if (!this._client.Utils.isNil(objectState.createOp)) { + this._mergeInitialDataFromCreateOperation(objectState.createOp); + } + } + + // if object got tombstoned, the update object will include all data that got cleared. + // otherwise it is a diff between previous value and new value from object state. + return this._updateFromDataDiff(previousDataRef, this._dataRef); + } + + /** + * @internal + */ + onGCInterval(): void { + // should remove any tombstoned entries from the underlying map data that have exceeded the GC grace period + + const keysToDelete: string[] = []; + for (const [key, value] of this._dataRef.data.entries()) { + if (value.tombstone === true && Date.now() - value.tombstonedAt! >= DEFAULTS.gcGracePeriod) { + keysToDelete.push(key); + } + } + + keysToDelete.forEach((x) => this._dataRef.data.delete(x)); + } + + protected _getZeroValueData(): LiveMapData { + return { data: new Map() }; + } + + protected _updateFromDataDiff(prevDataRef: LiveMapData, newDataRef: LiveMapData): LiveMapUpdate { + const update: LiveMapUpdate = { update: {} }; + + for (const [key, currentEntry] of prevDataRef.data.entries()) { + const typedKey: keyof T & string = key; + // any non-tombstoned properties that exist on a current map, but not in the new data - got removed + if (currentEntry.tombstone === false && !newDataRef.data.has(typedKey)) { + update.update[typedKey] = 'removed'; + } + } + + for (const [key, newEntry] of newDataRef.data.entries()) { + const typedKey: keyof T & string = key; + if (!prevDataRef.data.has(typedKey)) { + // if property does not exist in the current map, but new data has it as a non-tombstoned property - got updated + if (newEntry.tombstone === false) { + update.update[typedKey] = 'updated'; + continue; + } + + // otherwise, if new data has this prop tombstoned - do nothing, as property didn't exist anyway + if (newEntry.tombstone === true) { + continue; + } + } + + // properties that exist both in current and new map data need to have their values compared to decide on the update type + const currentEntry = prevDataRef.data.get(typedKey)!; + + // compare tombstones first + if (currentEntry.tombstone === true && newEntry.tombstone === false) { + // current prop is tombstoned, but new is not. it means prop was updated to a meaningful value + update.update[typedKey] = 'updated'; + continue; + } + if (currentEntry.tombstone === false && newEntry.tombstone === true) { + // current prop is not tombstoned, but new is. it means prop was removed + update.update[typedKey] = 'removed'; + continue; + } + if (currentEntry.tombstone === true && newEntry.tombstone === true) { + // both props are tombstoned - treat as noop, as there is no data to compare. + continue; + } + + // both props exist and are not tombstoned, need to compare values with deep equals to see if it was changed + const valueChanged = !dequal(currentEntry.data, newEntry.data); + if (valueChanged) { + update.update[typedKey] = 'updated'; + continue; + } + } + + return update; + } + + protected _mergeInitialDataFromCreateOperation(objectOperation: ObjectOperation): LiveMapUpdate { + if (this._client.Utils.isNil(objectOperation.map)) { + // if a map object is missing for the MAP_CREATE op, the initial value is implicitly an empty map. + // in this case there is nothing to merge into the current map, so we can just end processing the op. + return { update: {} }; + } + + const aggregatedUpdate: LiveMapUpdate = { update: {} }; + // in order to apply MAP_CREATE op for an existing map, we should merge their underlying entries keys. + // we can do this by iterating over entries from MAP_CREATE op and apply changes on per-key basis as if we had MAP_SET, MAP_REMOVE operations. + Object.entries(objectOperation.map.entries ?? {}).forEach(([key, entry]) => { + // for a MAP_CREATE operation we must use the serial value available on an entry, instead of a serial on a message + const opSerial = entry.timeserial; + let update: LiveMapUpdate | LiveObjectUpdateNoop; + if (entry.tombstone === true) { + // entry in MAP_CREATE op is removed, try to apply MAP_REMOVE op + update = this._applyMapRemove({ key }, opSerial); + } else { + // entry in MAP_CREATE op is not removed, try to set it via MAP_SET op + update = this._applyMapSet({ key, data: entry.data }, opSerial); + } + + // skip noop updates + if ((update as LiveObjectUpdateNoop).noop) { + return; + } + + // otherwise copy update data to aggregated update + Object.assign(aggregatedUpdate.update, update.update); + }); + + this._createOperationIsMerged = true; + + return aggregatedUpdate; + } + + private _throwNoPayloadError(op: ObjectOperation): void { + throw new this._client.ErrorInfo( + `No payload found for ${op.action} op for LiveMap objectId=${this.getObjectId()}`, + 92000, + 500, + ); + } + + private _applyMapCreate(op: ObjectOperation): LiveMapUpdate | LiveObjectUpdateNoop { + if (this._createOperationIsMerged) { + // There can't be two different create operation for the same object id, because the object id + // fully encodes that operation. This means we can safely ignore any new incoming create operations + // if we already merged it once. + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveMap._applyMapCreate()', + `skipping applying MAP_CREATE op on a map instance as it was already applied before; objectId=${this.getObjectId()}`, + ); + return { noop: true }; + } + + if (this._semantics !== op.map?.semantics) { + throw new this._client.ErrorInfo( + `Cannot apply MAP_CREATE op on LiveMap objectId=${this.getObjectId()}; map's semantics=${this._semantics}, but op expected ${op.map?.semantics}`, + 92000, + 500, + ); + } + + return this._mergeInitialDataFromCreateOperation(op); + } + + private _applyMapSet(op: MapOp, opSerial: string | undefined): LiveMapUpdate | LiveObjectUpdateNoop { + const { ErrorInfo, Utils } = this._client; + + const existingEntry = this._dataRef.data.get(op.key); + if (existingEntry && !this._canApplyMapOperation(existingEntry.timeserial, opSerial)) { + // the operation's serial <= the entry's serial, ignore the operation. + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveMap._applyMapSet()', + `skipping update for key="${op.key}": op serial ${opSerial?.toString()} <= entry serial ${existingEntry.timeserial?.toString()}; objectId=${this.getObjectId()}`, + ); + return { noop: true }; + } + + if ( + Utils.isNil(op.data) || + (Utils.isNil(op.data.objectId) && + Utils.isNil(op.data.boolean) && + Utils.isNil(op.data.bytes) && + Utils.isNil(op.data.number) && + Utils.isNil(op.data.string)) + ) { + throw new ErrorInfo( + `Invalid object data for MAP_SET op on objectId=${this.getObjectId()} on key=${op.key}`, + 92000, + 500, + ); + } + + let liveData: ObjectData; + if (!Utils.isNil(op.data.objectId)) { + liveData = { objectId: op.data.objectId } as ObjectIdObjectData; + // this MAP_SET op is setting a key to point to another object via its object id, + // but it is possible that we don't have the corresponding object in the pool yet (for example, we haven't seen the *_CREATE op for it). + // we don't want to return undefined from this map's .get() method even if we don't have the object, + // so instead we create a zero-value object for that object id if it not exists. + this._objects.getPool().createZeroValueObjectIfNotExists(op.data.objectId); + } else { + liveData = { + encoding: op.data.encoding, + boolean: op.data.boolean, + bytes: op.data.bytes, + number: op.data.number, + string: op.data.string, + } as ValueObjectData; + } + + if (existingEntry) { + existingEntry.tombstone = false; + existingEntry.tombstonedAt = undefined; + existingEntry.timeserial = opSerial; + existingEntry.data = liveData; + } else { + const newEntry: LiveMapEntry = { + tombstone: false, + tombstonedAt: undefined, + timeserial: opSerial, + data: liveData, + }; + this._dataRef.data.set(op.key, newEntry); + } + + const update: LiveMapUpdate = { update: {} }; + const typedKey: keyof T & string = op.key; + update.update[typedKey] = 'updated'; + + return update; + } + + private _applyMapRemove(op: MapOp, opSerial: string | undefined): LiveMapUpdate | LiveObjectUpdateNoop { + const existingEntry = this._dataRef.data.get(op.key); + if (existingEntry && !this._canApplyMapOperation(existingEntry.timeserial, opSerial)) { + // the operation's serial <= the entry's serial, ignore the operation. + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MICRO, + 'LiveMap._applyMapRemove()', + `skipping remove for key="${op.key}": op serial ${opSerial?.toString()} <= entry serial ${existingEntry.timeserial?.toString()}; objectId=${this.getObjectId()}`, + ); + return { noop: true }; + } + + if (existingEntry) { + existingEntry.tombstone = true; + existingEntry.tombstonedAt = Date.now(); + existingEntry.timeserial = opSerial; + existingEntry.data = undefined; + } else { + const newEntry: LiveMapEntry = { + tombstone: true, + tombstonedAt: Date.now(), + timeserial: opSerial, + data: undefined, + }; + this._dataRef.data.set(op.key, newEntry); + } + + const update: LiveMapUpdate = { update: {} }; + const typedKey: keyof T & string = op.key; + update.update[typedKey] = 'removed'; + + return update; + } + + /** + * Returns true if the serials of the given operation and entry indicate that + * the operation should be applied to the entry, following the CRDT semantics of this LiveMap. + */ + private _canApplyMapOperation(mapEntrySerial: string | undefined, opSerial: string | undefined): boolean { + // for LWW CRDT semantics (the only supported LiveMap semantic) an operation + // should only be applied if its serial is strictly greater ("after") than an entry's serial. + + if (!mapEntrySerial && !opSerial) { + // if both serials are nullish or empty strings, we treat them as the "earliest possible" serials, + // in which case they are "equal", so the operation should not be applied + return false; + } + + if (!mapEntrySerial) { + // any operation serial is greater than non-existing entry serial + return true; + } + + if (!opSerial) { + // non-existing operation serial is lower than any entry serial + return false; + } + + // if both serials exist, compare them lexicographically + return opSerial > mapEntrySerial; + } + + private _liveMapDataFromMapEntries(entries: Record): LiveMapData { + const liveMapData: LiveMapData = { + data: new Map(), + }; + + // need to iterate over entries to correctly process optional parameters + Object.entries(entries ?? {}).forEach(([key, entry]) => { + let liveData: ObjectData | undefined = undefined; + + if (!this._client.Utils.isNil(entry.data)) { + if (!this._client.Utils.isNil(entry.data.objectId)) { + liveData = { objectId: entry.data.objectId } as ObjectIdObjectData; + } else { + liveData = { + encoding: entry.data.encoding, + boolean: entry.data.boolean, + bytes: entry.data.bytes, + number: entry.data.number, + string: entry.data.string, + } as ValueObjectData; + } + } + + const liveDataEntry: LiveMapEntry = { + timeserial: entry.timeserial, + data: liveData, + // consider object as tombstoned only if we received an explicit flag stating that. otherwise it exists + tombstone: entry.tombstone === true, + tombstonedAt: entry.tombstone === true ? Date.now() : undefined, + }; + + liveMapData.data.set(key, liveDataEntry); + }); + + return liveMapData; + } + + /** + * Returns value as is if MapEntry stores a primitive type, or a reference to another LiveObject from the pool if it stores an objectId. + */ + private _getResolvedValueFromObjectData(data: ObjectData): PrimitiveObjectValue | LiveObject | undefined { + // if object data stores one of the primitive values, just return it as is. + const asValueObject = data as ValueObjectData; + if (asValueObject.boolean !== undefined) { + return asValueObject.boolean; + } + if (asValueObject.bytes !== undefined) { + return asValueObject.bytes; + } + if (asValueObject.number !== undefined) { + return asValueObject.number; + } + if (asValueObject.string !== undefined) { + return asValueObject.string; + } + + // otherwise, it has an objectId reference, and we should get the actual object from the pool + const objectId = (data as ObjectIdObjectData).objectId; + const refObject: LiveObject | undefined = this._objects.getPool().get(objectId); + if (!refObject) { + return undefined; + } + + if (refObject.isTombstoned()) { + // tombstoned objects must not be surfaced to the end users + return undefined; + } + + return refObject; + } + + private _isMapEntryTombstoned(entry: LiveMapEntry): boolean { + if (entry.tombstone === true) { + return true; + } + + // data always exists for non-tombstoned entries + const data = entry.data!; + if ('objectId' in data) { + const refObject = this._objects.getPool().get(data.objectId); + + if (refObject?.isTombstoned()) { + // entry that points to tombstoned object should be considered tombstoned as well + return true; + } + } + + return false; + } +} diff --git a/src/plugins/objects/liveobject.ts b/src/plugins/objects/liveobject.ts new file mode 100644 index 0000000000..d0870d5028 --- /dev/null +++ b/src/plugins/objects/liveobject.ts @@ -0,0 +1,257 @@ +import type BaseClient from 'common/lib/client/baseclient'; +import type EventEmitter from 'common/lib/util/eventemitter'; +import { ObjectMessage, ObjectOperation, ObjectState } from './objectmessage'; +import { Objects } from './objects'; + +export enum LiveObjectSubscriptionEvent { + updated = 'updated', +} + +export interface LiveObjectData { + data: any; +} + +export interface LiveObjectUpdate { + update: any; +} + +export interface LiveObjectUpdateNoop { + // have optional update field with undefined type so it's not possible to create a noop object with a meaningful update property. + update?: undefined; + noop: true; +} + +export interface SubscribeResponse { + unsubscribe(): void; +} + +export enum LiveObjectLifecycleEvent { + deleted = 'deleted', +} + +export type LiveObjectLifecycleEventCallback = () => void; + +export interface OnLiveObjectLifecycleEventResponse { + off(): void; +} + +export abstract class LiveObject< + TData extends LiveObjectData = LiveObjectData, + TUpdate extends LiveObjectUpdate = LiveObjectUpdate, +> { + protected _client: BaseClient; + protected _subscriptions: EventEmitter; + protected _lifecycleEvents: EventEmitter; + protected _objectId: string; + /** + * Represents an aggregated value for an object, which combines the initial value for an object from the create operation, + * and all object operations applied to the object. + */ + protected _dataRef: TData; + protected _siteTimeserials: Record; + protected _createOperationIsMerged: boolean; + private _tombstone: boolean; + /** + * Even though the {@link ObjectMessage.serial} value from the operation that deleted the object contains the timestamp value, + * the serial should be treated as an opaque string on the client, meaning we should not attempt to parse it. + * + * Therefore, we need to set our own timestamp using local clock when the object is deleted client-side. + * Strictly speaking, this does make an assumption about the client clock not being too heavily skewed behind the server, + * but it is an acceptable compromise for the time being, as the likelihood of encountering a race here is pretty low given the grace periods we use. + */ + private _tombstonedAt: number | undefined; + + protected constructor( + protected _objects: Objects, + objectId: string, + ) { + this._client = this._objects.getClient(); + this._subscriptions = new this._client.EventEmitter(this._client.logger); + this._lifecycleEvents = new this._client.EventEmitter(this._client.logger); + this._objectId = objectId; + this._dataRef = this._getZeroValueData(); + // use empty map of serials by default, so any future operation can be applied to this object + this._siteTimeserials = {}; + this._createOperationIsMerged = false; + this._tombstone = false; + } + + subscribe(listener: (update: TUpdate) => void): SubscribeResponse { + this._objects.throwIfInvalidAccessApiConfiguration(); + + this._subscriptions.on(LiveObjectSubscriptionEvent.updated, listener); + + const unsubscribe = () => { + this._subscriptions.off(LiveObjectSubscriptionEvent.updated, listener); + }; + + return { unsubscribe }; + } + + unsubscribe(listener: (update: TUpdate) => void): void { + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. + + // current implementation of the EventEmitter will remove all listeners if .off is called without arguments or with nullish arguments. + // or when called with just an event argument, it will remove all listeners for the event. + // thus we need to check that listener does actually exist before calling .off. + if (this._client.Utils.isNil(listener)) { + return; + } + + this._subscriptions.off(LiveObjectSubscriptionEvent.updated, listener); + } + + unsubscribeAll(): void { + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. + this._subscriptions.off(LiveObjectSubscriptionEvent.updated); + } + + on(event: LiveObjectLifecycleEvent, callback: LiveObjectLifecycleEventCallback): OnLiveObjectLifecycleEventResponse { + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. + this._lifecycleEvents.on(event, callback); + + const off = () => { + this._lifecycleEvents.off(event, callback); + }; + + return { off }; + } + + off(event: LiveObjectLifecycleEvent, callback: LiveObjectLifecycleEventCallback): void { + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. + + // prevent accidentally calling .off without any arguments on an EventEmitter and removing all callbacks + if (this._client.Utils.isNil(event) && this._client.Utils.isNil(callback)) { + return; + } + + this._lifecycleEvents.off(event, callback); + } + + offAll(): void { + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. + this._lifecycleEvents.off(); + } + + /** + * @internal + */ + getObjectId(): string { + return this._objectId; + } + + /** + * Emits the {@link LiveObjectSubscriptionEvent.updated} event with provided update object if it isn't a noop. + * + * @internal + */ + notifyUpdated(update: TUpdate | LiveObjectUpdateNoop): void { + // should not emit update event if update was noop + if ((update as LiveObjectUpdateNoop).noop) { + return; + } + + this._subscriptions.emit(LiveObjectSubscriptionEvent.updated, update); + } + + /** + * Clears the object's data, cancels any buffered operations and sets the tombstone flag to `true`. + * + * @internal + */ + tombstone(): TUpdate { + this._tombstone = true; + this._tombstonedAt = Date.now(); + const update = this.clearData(); + this._lifecycleEvents.emit(LiveObjectLifecycleEvent.deleted); + + return update; + } + + /** + * @internal + */ + isTombstoned(): boolean { + return this._tombstone; + } + + /** + * @internal + */ + tombstonedAt(): number | undefined { + return this._tombstonedAt; + } + + /** + * @internal + */ + clearData(): TUpdate { + const previousDataRef = this._dataRef; + this._dataRef = this._getZeroValueData(); + return this._updateFromDataDiff(previousDataRef, this._dataRef); + } + + /** + * Returns true if the given serial indicates that the operation to which it belongs should be applied to the object. + * + * An operation should be applied if its serial is strictly greater than the serial in the `siteTimeserials` map for the same site. + * If `siteTimeserials` map does not contain a serial for the same site, the operation should be applied. + */ + protected _canApplyOperation(opSerial: string | undefined, opSiteCode: string | undefined): boolean { + if (!opSerial) { + throw new this._client.ErrorInfo(`Invalid serial: ${opSerial}`, 92000, 500); + } + + if (!opSiteCode) { + throw new this._client.ErrorInfo(`Invalid site code: ${opSiteCode}`, 92000, 500); + } + + const siteSerial = this._siteTimeserials[opSiteCode]; + return !siteSerial || opSerial > siteSerial; + } + + protected _applyObjectDelete(): TUpdate { + return this.tombstone(); + } + + /** + * Apply object operation message on this LiveObject. + * + * @internal + */ + abstract applyOperation(op: ObjectOperation, msg: ObjectMessage): void; + /** + * Overrides internal data for this LiveObject with data from the given object state. + * Provided object state should hold a valid data for current LiveObject, e.g. counter data for LiveCounter, map data for LiveMap. + * + * Object states are received during sync sequence, and sync sequence is a source of truth for the current state of the objects, + * so we can use the data received from the sync sequence directly and override any data values or site serials this LiveObject has + * without the need to merge them. + * + * Returns an update object that describes the changes applied based on the object's previous value. + * + * @internal + */ + abstract overrideWithObjectState(objectState: ObjectState): TUpdate | LiveObjectUpdateNoop; + /** + * @internal + */ + abstract onGCInterval(): void; + + protected abstract _getZeroValueData(): TData; + /** + * Calculate the update object based on the current LiveObject data and incoming new data. + */ + protected abstract _updateFromDataDiff(prevDataRef: TData, newDataRef: TData): TUpdate; + /** + * Merges the initial data from the create operation into the LiveObject. + * + * Client SDKs do not need to keep around the object operation that created the object, + * so we can merge the initial data the first time we receive it for the object, + * and work with aggregated value after that. + * + * This saves us from needing to merge the initial value with operations applied to + * the object every time the object is read. + */ + protected abstract _mergeInitialDataFromCreateOperation(objectOperation: ObjectOperation): TUpdate; +} diff --git a/src/plugins/objects/objectid.ts b/src/plugins/objects/objectid.ts new file mode 100644 index 0000000000..77508f43f1 --- /dev/null +++ b/src/plugins/objects/objectid.ts @@ -0,0 +1,69 @@ +import type BaseClient from 'common/lib/client/baseclient'; +import type Platform from 'common/platform'; +import type { Bufferlike } from 'common/platform'; + +export type LiveObjectType = 'map' | 'counter'; + +/** + * Represents a parsed object id. + * + * @internal + */ +export class ObjectId { + private constructor( + readonly type: LiveObjectType, + readonly hash: string, + readonly msTimestamp: number, + ) {} + + static fromInitialValue( + platform: typeof Platform, + objectType: LiveObjectType, + encodedInitialValue: Bufferlike, + nonce: string, + msTimestamp: number, + ): ObjectId { + const valueForHashBuffer = platform.BufferUtils.concat([ + encodedInitialValue, + platform.BufferUtils.utf8Encode(':'), + platform.BufferUtils.utf8Encode(nonce), + ]); + const hashBuffer = platform.BufferUtils.sha256(valueForHashBuffer); + const hash = platform.BufferUtils.base64UrlEncode(hashBuffer); + + return new ObjectId(objectType, hash, msTimestamp); + } + + /** + * Create ObjectId instance from hashed object id string. + */ + static fromString(client: BaseClient, objectId: string | null | undefined): ObjectId { + if (client.Utils.isNil(objectId)) { + throw new client.ErrorInfo('Invalid object id string', 92000, 500); + } + + const [type, rest] = objectId.split(':'); + if (!type || !rest) { + throw new client.ErrorInfo('Invalid object id string', 92000, 500); + } + + if (!['map', 'counter'].includes(type)) { + throw new client.ErrorInfo(`Invalid object type in object id: ${objectId}`, 92000, 500); + } + + const [hash, msTimestamp] = rest.split('@'); + if (!hash || !msTimestamp) { + throw new client.ErrorInfo('Invalid object id string', 92000, 500); + } + + if (!Number.isInteger(Number.parseInt(msTimestamp))) { + throw new client.ErrorInfo('Invalid object id string', 92000, 500); + } + + return new ObjectId(type as LiveObjectType, hash, Number.parseInt(msTimestamp)); + } + + toString(): string { + return `${this.type}:${this.hash}@${this.msTimestamp}`; + } +} diff --git a/src/plugins/objects/objectmessage.ts b/src/plugins/objects/objectmessage.ts new file mode 100644 index 0000000000..f5f2abe4e4 --- /dev/null +++ b/src/plugins/objects/objectmessage.ts @@ -0,0 +1,653 @@ +import type BaseClient from 'common/lib/client/baseclient'; +import type { MessageEncoding } from 'common/lib/types/basemessage'; +import type Logger from 'common/lib/util/logger'; +import type * as Utils from 'common/lib/util/utils'; +import type { Bufferlike } from 'common/platform'; + +export type EncodeInitialValueFunction = ( + data: any, + encoding?: string | null, +) => { data: any; encoding?: string | null }; + +export type EncodeObjectDataFunction = (data: ObjectData) => ObjectData; + +export enum ObjectOperationAction { + MAP_CREATE = 0, + MAP_SET = 1, + MAP_REMOVE = 2, + COUNTER_CREATE = 3, + COUNTER_INC = 4, + OBJECT_DELETE = 5, +} + +export enum MapSemantics { + LWW = 0, +} + +/** An ObjectData represents a value in an object on a channel. */ +export interface ObjectData { + /** A reference to another object, used to support composable object structures. */ + objectId?: string; + + /** Can be set by the client to indicate that value in `string` or `bytes` field have an encoding. */ + encoding?: string; + /** A primitive boolean leaf value in the object graph. Only one value field can be set. */ + boolean?: boolean; + /** A primitive binary leaf value in the object graph. Only one value field can be set. */ + bytes?: Bufferlike; + /** A primitive number leaf value in the object graph. Only one value field can be set. */ + number?: number; + /** A primitive string leaf value in the object graph. Only one value field can be set. */ + string?: string; +} + +/** A MapOp describes an operation to be applied to a Map object. */ +export interface MapOp { + /** The key of the map entry to which the operation should be applied. */ + key: string; + /** The data that the map entry should contain if the operation is a MAP_SET operation. */ + data?: ObjectData; +} + +/** A CounterOp describes an operation to be applied to a Counter object. */ +export interface CounterOp { + /** The data value that should be added to the counter */ + amount: number; +} + +/** A MapEntry represents the value at a given key in a Map object. */ +export interface MapEntry { + /** Indicates whether the map entry has been removed. */ + tombstone?: boolean; + /** + * The {@link ObjectMessage.serial} value of the last operation that was applied to the map entry. + * + * It is optional in a MAP_CREATE operation and might be missing, in which case the client should use a nullish value for it + * and treat it as the "earliest possible" serial for comparison purposes. + */ + timeserial?: string; + /** The data that represents the value of the map entry. */ + data?: ObjectData; +} + +/** An ObjectMap object represents a map of key-value pairs. */ +export interface ObjectMap { + /** The conflict-resolution semantics used by the map object. */ + semantics?: MapSemantics; + // The map entries, indexed by key. + entries?: Record; +} + +/** An ObjectCounter object represents an incrementable and decrementable value */ +export interface ObjectCounter { + /** The value of the counter */ + count?: number; +} + +/** An ObjectOperation describes an operation to be applied to an object on a channel. */ +export interface ObjectOperation { + /** Defines the operation to be applied to the object. */ + action: ObjectOperationAction; + /** The object ID of the object on a channel to which the operation should be applied. */ + objectId: string; + /** The payload for the operation if it is an operation on a Map object type. */ + mapOp?: MapOp; + /** The payload for the operation if it is an operation on a Counter object type. */ + counterOp?: CounterOp; + /** + * The payload for the operation if the operation is MAP_CREATE. + * Defines the initial value for the Map object. + */ + map?: ObjectMap; + /** + * The payload for the operation if the operation is COUNTER_CREATE. + * Defines the initial value for the Counter object. + */ + counter?: ObjectCounter; + /** + * The nonce, must be present on create operations. This is the random part + * that has been hashed with the type and initial value to create the object ID. + */ + nonce?: string; + /** + * The initial value bytes for the object. These bytes should be used along with the nonce + * and timestamp to create the object ID. Frontdoor will use this to verify the object ID. + * After verification the bytes will be decoded into the Map or Counter objects and + * the initialValue, nonce, and initialValueEncoding will be removed. + */ + initialValue?: Bufferlike; + /** The initial value encoding defines how the initialValue should be interpreted. */ + initialValueEncoding?: Utils.Format; +} + +/** An ObjectState describes the instantaneous state of an object on a channel. */ +export interface ObjectState { + /** The identifier of the object. */ + objectId: string; + /** A map of serials keyed by a {@link ObjectMessage.siteCode}, representing the last operations applied to this object */ + siteTimeserials: Record; + /** True if the object has been tombstoned. */ + tombstone: boolean; + /** + * The operation that created the object. + * + * Can be missing if create operation for the object is not known at this point. + */ + createOp?: ObjectOperation; + /** + * The data that represents the result of applying all operations to a Map object + * excluding the initial value from the create operation if it is a Map object type. + */ + map?: ObjectMap; + /** + * The data that represents the result of applying all operations to a Counter object + * excluding the initial value from the create operation if it is a Counter object type. + */ + counter?: ObjectCounter; +} + +// TODO: tidy up encoding/decoding logic for ObjectMessage: +// Should have separate WireObjectMessage with the correct types received from the server, do the necessary encoding/decoding there. +// For reference, see WireMessage and WirePresenceMessage +/** + * @internal + */ +export class ObjectMessage { + id?: string; + timestamp?: number; + clientId?: string; + connectionId?: string; + extras?: any; + /** + * Describes an operation to be applied to an object. + * + * Mutually exclusive with the `object` field. This field is only set on object messages if the `action` field of the `ProtocolMessage` encapsulating it is `OBJECT`. + */ + operation?: ObjectOperation; + /** + * Describes the instantaneous state of an object. + * + * Mutually exclusive with the `operation` field. This field is only set on object messages if the `action` field of the `ProtocolMessage` encapsulating it is `OBJECT_SYNC`. + */ + object?: ObjectState; + /** An opaque string that uniquely identifies this object message. */ + serial?: string; + /** An opaque string used as a key to update the map of serial values on an object. */ + siteCode?: string; + + constructor( + private _utils: typeof Utils, + private _messageEncoding: typeof MessageEncoding, + ) {} + + /** + * Protocol agnostic encoding of the object message's data entries. + * Mutates the provided ObjectMessage. + * + * Uses encoding functions from regular `Message` processing. + */ + static encode(message: ObjectMessage, client: BaseClient): ObjectMessage { + const encodeInitialValueFn: EncodeInitialValueFunction = (data, encoding) => { + const isNativeDataType = + typeof data == 'string' || + typeof data == 'number' || + typeof data == 'boolean' || + client.Platform.BufferUtils.isBuffer(data) || + data === null || + data === undefined; + + const { data: encodedData, encoding: newEncoding } = client.MessageEncoding.encodeData( + data, + encoding, + isNativeDataType, + ); + + return { + data: encodedData, + encoding: newEncoding, + }; + }; + + const encodeObjectDataFn: EncodeObjectDataFunction = (data) => { + // TODO: support encoding JSON objects as a JSON string on "string" property with an encoding of "json" + // https://ably.atlassian.net/browse/PUB-1667 + // for now just return values as they are + + return data; + }; + + message.operation = message.operation + ? ObjectMessage._encodeObjectOperation(message.operation, encodeObjectDataFn, encodeInitialValueFn) + : undefined; + message.object = message.object + ? ObjectMessage._encodeObjectState(message.object, encodeObjectDataFn, encodeInitialValueFn) + : undefined; + + return message; + } + + /** + * Mutates the provided ObjectMessage and decodes all data entries in the message. + * + * Format is used to decode the bytes value as it's implicitly encoded depending on the protocol used: + * - json: bytes are base64 encoded string + * - msgpack: bytes have a binary representation and don't need to be decoded + */ + static async decode( + message: ObjectMessage, + client: BaseClient, + logger: Logger, + LoggerClass: typeof Logger, + utils: typeof Utils, + format: Utils.Format | undefined, + ): Promise { + // TODO: decide how to handle individual errors from decoding values. currently we throw first ever error we get + + try { + if (message.object?.map?.entries) { + await ObjectMessage._decodeMapEntries(message.object.map.entries, client, format); + } + + if (message.object?.createOp?.map?.entries) { + await ObjectMessage._decodeMapEntries(message.object.createOp.map.entries, client, format); + } + + if (message.object?.createOp?.mapOp?.data) { + await ObjectMessage._decodeObjectData(message.object.createOp.mapOp.data, client, format); + } + + if (message.operation?.map?.entries) { + await ObjectMessage._decodeMapEntries(message.operation.map.entries, client, format); + } + + if (message.operation?.mapOp?.data) { + await ObjectMessage._decodeObjectData(message.operation.mapOp.data, client, format); + } + } catch (error) { + LoggerClass.logAction(logger, LoggerClass.LOG_ERROR, 'ObjectMessage.decode()', utils.inspectError(error)); + } + } + + static fromValues( + values: ObjectMessage | Record, + utils: typeof Utils, + messageEncoding: typeof MessageEncoding, + ): ObjectMessage { + return Object.assign(new ObjectMessage(utils, messageEncoding), values); + } + + static fromValuesArray( + values: (ObjectMessage | Record)[], + utils: typeof Utils, + messageEncoding: typeof MessageEncoding, + ): ObjectMessage[] { + const count = values.length; + const result = new Array(count); + + for (let i = 0; i < count; i++) { + result[i] = ObjectMessage.fromValues(values[i], utils, messageEncoding); + } + + return result; + } + + static encodeInitialValue( + initialValue: Partial, + client: BaseClient, + ): { + encodedInitialValue: Bufferlike; + format: Utils.Format; + } { + const format = client.options.useBinaryProtocol ? client.Utils.Format.msgpack : client.Utils.Format.json; + + // initial value object may contain user provided data that requires an additional encoding (for example buffers as map keys). + // so we need to encode that data first as if we were sending it over the wire. we can use an ObjectMessage methods for this + const msg = ObjectMessage.fromValues({ operation: initialValue }, client.Utils, client.MessageEncoding); + ObjectMessage.encode(msg, client); + const { operation: initialValueWithDataEncoding } = ObjectMessage._encodeForWireProtocol( + msg, + client.MessageEncoding, + format, + ); + + // initial value field should be represented as an array of bytes over the wire. so we encode the whole object based on the client encoding format + const encodedInitialValue = client.Utils.encodeBody(initialValueWithDataEncoding, client._MsgPack, format); + + // if we've got string result (for example, json encoding was used), we need to additionally convert it to bytes array with utf8 encoding + if (typeof encodedInitialValue === 'string') { + return { + encodedInitialValue: client.Platform.BufferUtils.utf8Encode(encodedInitialValue), + format, + }; + } + + return { + encodedInitialValue, + format, + }; + } + + private static async _decodeMapEntries( + mapEntries: Record, + client: BaseClient, + format: Utils.Format | undefined, + ): Promise { + for (const entry of Object.values(mapEntries)) { + if (entry.data) { + await ObjectMessage._decodeObjectData(entry.data, client, format); + } + } + } + + private static async _decodeObjectData( + objectData: ObjectData, + client: BaseClient, + format: Utils.Format | undefined, + ): Promise { + // TODO: support decoding JSON objects stored as a JSON string with an encoding of "json" + // https://ably.atlassian.net/browse/PUB-1667 + // currently we check only the "bytes" field: + // - if connection is msgpack - "bytes" was received as msgpack encoded bytes, no need to decode, it's already a buffer + // - if connection is json - "bytes" was received as a base64 string, need to decode it to a buffer + + if (format !== 'msgpack' && objectData.bytes != null) { + // connection is using JSON protocol, decode bytes value + objectData.bytes = client.Platform.BufferUtils.base64Decode(String(objectData.bytes)); + } + } + + private static _encodeObjectOperation( + objectOperation: ObjectOperation, + encodeObjectDataFn: EncodeObjectDataFunction, + encodeInitialValueFn: EncodeInitialValueFunction, + ): ObjectOperation { + // deep copy "objectOperation" object so we can modify the copy here. + // buffer values won't be correctly copied, so we will need to set them again explicitly. + const objectOperationCopy = JSON.parse(JSON.stringify(objectOperation)) as ObjectOperation; + + if (objectOperationCopy.mapOp?.data) { + // use original "objectOperation" object when encoding values, so we have access to the original buffer values. + objectOperationCopy.mapOp.data = ObjectMessage._encodeObjectData( + objectOperation.mapOp?.data!, + encodeObjectDataFn, + ); + } + + if (objectOperationCopy.map?.entries) { + Object.entries(objectOperationCopy.map.entries).forEach(([key, entry]) => { + if (entry.data) { + // use original "objectOperation" object when encoding values, so we have access to original buffer values. + entry.data = ObjectMessage._encodeObjectData(objectOperation?.map?.entries?.[key].data!, encodeObjectDataFn); + } + }); + } + + if (objectOperation.initialValue) { + // use original "objectOperation" object so we have access to the original buffer value + const { data: encodedInitialValue } = encodeInitialValueFn(objectOperation.initialValue); + objectOperationCopy.initialValue = encodedInitialValue; + } + + return objectOperationCopy; + } + + private static _encodeObjectState( + objectState: ObjectState, + encodeObjectDataFn: EncodeObjectDataFunction, + encodeInitialValueFn: EncodeInitialValueFunction, + ): ObjectState { + // deep copy "objectState" object so we can modify the copy here. + // buffer values won't be correctly copied, so we will need to set them again explicitly. + const objectStateCopy = JSON.parse(JSON.stringify(objectState)) as ObjectState; + + if (objectStateCopy.map?.entries) { + Object.entries(objectStateCopy.map.entries).forEach(([key, entry]) => { + if (entry.data) { + // use original "objectState" object when encoding values, so we have access to original buffer values. + entry.data = ObjectMessage._encodeObjectData(objectState?.map?.entries?.[key].data!, encodeObjectDataFn); + } + }); + } + + if (objectStateCopy.createOp) { + // use original "objectState" object when encoding values, so we have access to original buffer values. + objectStateCopy.createOp = ObjectMessage._encodeObjectOperation( + objectState.createOp!, + encodeObjectDataFn, + encodeInitialValueFn, + ); + } + + return objectStateCopy; + } + + private static _encodeObjectData(data: ObjectData, encodeFn: EncodeObjectDataFunction): ObjectData { + const encodedData = encodeFn(data); + return encodedData; + } + + /** + * Encodes operation and object fields of the ObjectMessage. Does not mutate the provided ObjectMessage. + * + * Uses encoding functions from regular `Message` processing. + */ + private static _encodeForWireProtocol( + message: ObjectMessage, + messageEncoding: typeof MessageEncoding, + format: Utils.Format, + ): { + operation?: ObjectOperation; + objectState?: ObjectState; + } { + const encodeInitialValueFn: EncodeInitialValueFunction = (data, encoding) => { + const { data: encodedData, encoding: newEncoding } = messageEncoding.encodeDataForWire(data, encoding, format); + return { + data: encodedData, + encoding: newEncoding, + }; + }; + + const encodeObjectDataFn: EncodeObjectDataFunction = (data) => { + // TODO: support encoding JSON objects as a JSON string on "string" property with an encoding of "json" + // https://ably.atlassian.net/browse/PUB-1667 + // currently we check only the "bytes" field: + // - if connection is msgpack - "bytes" will will be sent as msgpack bytes, no need to encode here + // - if connection is json - "bytes" will be encoded as a base64 string + + let encodedBytes: any = data.bytes; + if (data.bytes != null) { + const result = messageEncoding.encodeDataForWire(data.bytes, data.encoding, format); + encodedBytes = result.data; + // no need to change the encoding + } + + return { + ...data, + bytes: encodedBytes, + }; + }; + + const encodedOperation = message.operation + ? ObjectMessage._encodeObjectOperation(message.operation, encodeObjectDataFn, encodeInitialValueFn) + : undefined; + const encodedObjectState = message.object + ? ObjectMessage._encodeObjectState(message.object, encodeObjectDataFn, encodeInitialValueFn) + : undefined; + + return { + operation: encodedOperation, + objectState: encodedObjectState, + }; + } + + /** + * Overload toJSON() to intercept JSON.stringify(). + * + * This will prepare the message to be transmitted over the wire to Ably. + * It will encode the data payload according to the wire protocol used on the client. + * It will transform any client-side enum string representations into their corresponding numbers, if needed (like "action" fields). + */ + toJSON(): { + id?: string; + clientId?: string; + operation?: ObjectOperation; + object?: ObjectState; + extras?: any; + } { + // we can infer the format used by client by inspecting with what arguments this method was called. + // if JSON protocol is being used, the JSON.stringify() will be called and this toJSON() method will have a non-empty arguments list. + // MSGPack protocol implementation also calls toJSON(), but with an empty arguments list. + const format = arguments.length > 0 ? this._utils.Format.json : this._utils.Format.msgpack; + const { operation, objectState } = ObjectMessage._encodeForWireProtocol(this, this._messageEncoding, format); + + return { + id: this.id, + clientId: this.clientId, + operation, + object: objectState, + extras: this.extras, + }; + } + + toString(): string { + let result = '[ObjectMessage'; + + if (this.id) result += '; id=' + this.id; + if (this.timestamp) result += '; timestamp=' + this.timestamp; + if (this.clientId) result += '; clientId=' + this.clientId; + if (this.connectionId) result += '; connectionId=' + this.connectionId; + // TODO: prettify output for operation and object and encode buffers. + // see examples for data in Message and PresenceMessage + if (this.operation) result += '; operation=' + JSON.stringify(this.operation); + if (this.object) result += '; object=' + JSON.stringify(this.object); + if (this.extras) result += '; extras=' + JSON.stringify(this.extras); + if (this.serial) result += '; serial=' + this.serial; + if (this.siteCode) result += '; siteCode=' + this.siteCode; + + result += ']'; + + return result; + } + + getMessageSize(): number { + let size = 0; + + size += this.clientId?.length ?? 0; + if (this.operation) { + size += this._getObjectOperationSize(this.operation); + } + if (this.object) { + size += this._getObjectStateSize(this.object); + } + if (this.extras) { + size += JSON.stringify(this.extras).length; + } + + return size; + } + + private _getObjectOperationSize(operation: ObjectOperation): number { + let size = 0; + + if (operation.mapOp) { + size += this._getMapOpSize(operation.mapOp); + } + if (operation.counterOp) { + size += this._getCounterOpSize(operation.counterOp); + } + if (operation.map) { + size += this._getObjectMapSize(operation.map); + } + if (operation.counter) { + size += this._getObjectCounterSize(operation.counter); + } + + return size; + } + + private _getObjectStateSize(obj: ObjectState): number { + let size = 0; + + if (obj.map) { + size += this._getObjectMapSize(obj.map); + } + if (obj.counter) { + size += this._getObjectCounterSize(obj.counter); + } + if (obj.createOp) { + size += this._getObjectOperationSize(obj.createOp); + } + + return size; + } + + private _getObjectMapSize(map: ObjectMap): number { + let size = 0; + + Object.entries(map.entries ?? {}).forEach(([key, entry]) => { + size += key?.length ?? 0; + if (entry) { + size += this._getMapEntrySize(entry); + } + }); + + return size; + } + + private _getObjectCounterSize(counter: ObjectCounter): number { + if (counter.count == null) { + return 0; + } + + return 8; + } + + private _getMapEntrySize(entry: MapEntry): number { + let size = 0; + + if (entry.data) { + size += this._getObjectDataSize(entry.data); + } + + return size; + } + + private _getMapOpSize(mapOp: MapOp): number { + let size = 0; + + size += mapOp.key?.length ?? 0; + + if (mapOp.data) { + size += this._getObjectDataSize(mapOp.data); + } + + return size; + } + + private _getCounterOpSize(operation: CounterOp): number { + if (operation.amount == null) { + return 0; + } + + return 8; + } + + private _getObjectDataSize(data: ObjectData): number { + let size = 0; + + if (data.boolean != null) { + size += this._utils.dataSizeBytes(data.boolean); + } + if (data.bytes != null) { + size += this._utils.dataSizeBytes(data.bytes); + } + if (data.number != null) { + size += this._utils.dataSizeBytes(data.number); + } + if (data.string != null) { + size += this._utils.dataSizeBytes(data.string); + } + + return size; + } +} diff --git a/src/plugins/objects/objects.ts b/src/plugins/objects/objects.ts new file mode 100644 index 0000000000..45223cee8e --- /dev/null +++ b/src/plugins/objects/objects.ts @@ -0,0 +1,508 @@ +import type BaseClient from 'common/lib/client/baseclient'; +import type RealtimeChannel from 'common/lib/client/realtimechannel'; +import type EventEmitter from 'common/lib/util/eventemitter'; +import type * as API from '../../../ably'; +import { BatchContext } from './batchcontext'; +import { DEFAULTS } from './defaults'; +import { LiveCounter } from './livecounter'; +import { LiveMap } from './livemap'; +import { LiveObject, LiveObjectUpdate, LiveObjectUpdateNoop } from './liveobject'; +import { ObjectMessage, ObjectOperationAction } from './objectmessage'; +import { ObjectsPool, ROOT_OBJECT_ID } from './objectspool'; +import { SyncObjectsDataPool } from './syncobjectsdatapool'; + +export enum ObjectsEvent { + syncing = 'syncing', + synced = 'synced', +} + +export enum ObjectsState { + initialized = 'initialized', + syncing = 'syncing', + synced = 'synced', +} + +const StateToEventsMap: Record = { + initialized: undefined, + syncing: ObjectsEvent.syncing, + synced: ObjectsEvent.synced, +}; + +export type ObjectsEventCallback = () => void; + +export interface OnObjectsEventResponse { + off(): void; +} + +export type BatchCallback = (batchContext: BatchContext) => void; + +export class Objects { + private _client: BaseClient; + private _channel: RealtimeChannel; + private _state: ObjectsState; + // composition over inheritance since we cannot import class directly into plugin code. + // instead we obtain a class type from the client + private _eventEmitterInternal: EventEmitter; + // related to RTC10, should have a separate EventEmitter for users of the library + private _eventEmitterPublic: EventEmitter; + private _objectsPool: ObjectsPool; + private _syncObjectsDataPool: SyncObjectsDataPool; + private _currentSyncId: string | undefined; + private _currentSyncCursor: string | undefined; + private _bufferedObjectOperations: ObjectMessage[]; + + // Used by tests + static _DEFAULTS = DEFAULTS; + + constructor(channel: RealtimeChannel) { + this._channel = channel; + this._client = channel.client; + this._state = ObjectsState.initialized; + this._eventEmitterInternal = new this._client.EventEmitter(this._client.logger); + this._eventEmitterPublic = new this._client.EventEmitter(this._client.logger); + this._objectsPool = new ObjectsPool(this); + this._syncObjectsDataPool = new SyncObjectsDataPool(this); + this._bufferedObjectOperations = []; + } + + /** + * When called without a type variable, we return a default root type which is based on globally defined interface for Objects feature. + * A user can provide an explicit type for the getRoot method to explicitly set the type structure on this particular channel. + * This is useful when working with multiple channels with different underlying data structure. + */ + async getRoot(): Promise> { + this.throwIfInvalidAccessApiConfiguration(); + + // if we're not synced yet, wait for sync sequence to finish before returning root + if (this._state !== ObjectsState.synced) { + await this._eventEmitterInternal.once(ObjectsEvent.synced); + } + + return this._objectsPool.get(ROOT_OBJECT_ID) as LiveMap; + } + + /** + * Provides access to the synchronous write API for Objects that can be used to batch multiple operations together in a single channel message. + */ + async batch(callback: BatchCallback): Promise { + this.throwIfInvalidWriteApiConfiguration(); + + const root = await this.getRoot(); + const context = new BatchContext(this, root); + + try { + callback(context); + await context.flush(); + } finally { + context.close(); + } + } + + /** + * Send a MAP_CREATE operation to the realtime system to create a new map object in the pool. + * + * Once the ACK message is received, the method returns the object from the local pool if it got created due to + * the echoed MAP_CREATE operation, or if it wasn't received yet, the method creates a new object locally using the provided data and returns it. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with an object containing provided data. + */ + async createMap(entries?: T): Promise> { + this.throwIfInvalidWriteApiConfiguration(); + + const msg = await LiveMap.createMapCreateMessage(this, entries); + const objectId = msg.operation?.objectId!; + + await this.publish([msg]); + + // we may have already received the MAP_CREATE operation at this point, as it could arrive before the ACK for our publish message. + // this means the object might already exist in the local pool, having been added during the usual MAP_CREATE operation process. + // here we check if the object is present, and return it if found; otherwise, create a new object on the client side. + if (this._objectsPool.get(objectId)) { + return this._objectsPool.get(objectId) as LiveMap; + } + + // we haven't received the MAP_CREATE operation yet, so we can create a new map object using the locally constructed object operation. + // we don't know the serials for map entries, so we assign an "earliest possible" serial to each entry, so that any subsequent operation can be applied to them. + // we mark the MAP_CREATE operation as merged for the object, guaranteeing its idempotency and preventing it from being applied again when the operation arrives. + const map = LiveMap.fromObjectOperation(this, msg.operation!); + this._objectsPool.set(objectId, map); + + return map; + } + + /** + * Send a COUNTER_CREATE operation to the realtime system to create a new counter object in the pool. + * + * Once the ACK message is received, the method returns the object from the local pool if it got created due to + * the echoed COUNTER_CREATE operation, or if it wasn't received yet, the method creates a new object locally using the provided data and returns it. + * + * @returns A promise which resolves upon receiving the ACK message for the published operation message. A promise is resolved with an object containing provided data. + */ + async createCounter(count?: number): Promise { + this.throwIfInvalidWriteApiConfiguration(); + + const msg = await LiveCounter.createCounterCreateMessage(this, count); + const objectId = msg.operation?.objectId!; + + await this.publish([msg]); + + // we may have already received the COUNTER_CREATE operation at this point, as it could arrive before the ACK for our publish message. + // this means the object might already exist in the local pool, having been added during the usual COUNTER_CREATE operation process. + // here we check if the object is present, and return it if found; otherwise, create a new object on the client side. + if (this._objectsPool.get(objectId)) { + return this._objectsPool.get(objectId) as LiveCounter; + } + + // we haven't received the COUNTER_CREATE operation yet, so we can create a new counter object using the locally constructed object operation. + // we mark the COUNTER_CREATE operation as merged for the object, guaranteeing its idempotency. this ensures we don't double count the initial counter value when the operation arrives. + const counter = LiveCounter.fromObjectOperation(this, msg.operation!); + this._objectsPool.set(objectId, counter); + + return counter; + } + + on(event: ObjectsEvent, callback: ObjectsEventCallback): OnObjectsEventResponse { + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. + this._eventEmitterPublic.on(event, callback); + + const off = () => { + this._eventEmitterPublic.off(event, callback); + }; + + return { off }; + } + + off(event: ObjectsEvent, callback: ObjectsEventCallback): void { + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. + + // prevent accidentally calling .off without any arguments on an EventEmitter and removing all callbacks + if (this._client.Utils.isNil(event) && this._client.Utils.isNil(callback)) { + return; + } + + this._eventEmitterPublic.off(event, callback); + } + + offAll(): void { + // this public API method can be called without specific configuration, so checking for invalid settings is unnecessary. + this._eventEmitterPublic.off(); + } + + /** + * @internal + */ + getPool(): ObjectsPool { + return this._objectsPool; + } + + /** + * @internal + */ + getChannel(): RealtimeChannel { + return this._channel; + } + + /** + * @internal + */ + getClient(): BaseClient { + return this._client; + } + + /** + * @internal + */ + handleObjectSyncMessages(objectMessages: ObjectMessage[], syncChannelSerial: string | null | undefined): void { + const { syncId, syncCursor } = this._parseSyncChannelSerial(syncChannelSerial); + const newSyncSequence = this._currentSyncId !== syncId; + if (newSyncSequence) { + this._startNewSync(syncId, syncCursor); + } + + this._syncObjectsDataPool.applyObjectSyncMessages(objectMessages); + + // if this is the last (or only) message in a sequence of sync updates, end the sync + if (!syncCursor) { + // defer the state change event until the next tick if this was a new sync sequence + // to allow any event listeners to process the start of the new sequence event that was emitted earlier during this event loop. + this._endSync(newSyncSequence); + } + } + + /** + * @internal + */ + handleObjectMessages(objectMessages: ObjectMessage[]): void { + if (this._state !== ObjectsState.synced) { + // The client receives object messages in realtime over the channel concurrently with the sync sequence. + // Some of the incoming object messages may have already been applied to the objects described in + // the sync sequence, but others may not; therefore we must buffer these messages so that we can apply + // them to the objects once the sync is complete. + this._bufferedObjectOperations.push(...objectMessages); + return; + } + + this._applyObjectMessages(objectMessages); + } + + /** + * @internal + */ + onAttached(hasObjects?: boolean): void { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MINOR, + 'Objects.onAttached()', + `channel=${this._channel.name}, hasObjects=${hasObjects}`, + ); + + const fromInitializedState = this._state === ObjectsState.initialized; + if (hasObjects || fromInitializedState) { + // should always start a new sync sequence if we're in the initialized state, no matter the HAS_OBJECTS flag value. + // this guarantees we emit both "syncing" -> "synced" events in that order. + this._startNewSync(); + } + + if (!hasObjects) { + // if no HAS_OBJECTS flag received on attach, we can end sync sequence immediately and treat it as no objects on a channel. + // reset the objects pool to its initial state, and emit update events so subscribers to root object get notified about changes. + this._objectsPool.resetToInitialPool(true); + this._syncObjectsDataPool.clear(); + // defer the state change event until the next tick if we started a new sequence just now due to being in initialized state. + // this allows any event listeners to process the start of the new sequence event that was emitted earlier during this event loop. + this._endSync(fromInitializedState); + } + } + + /** + * @internal + */ + actOnChannelState(state: API.ChannelState, hasObjects?: boolean): void { + switch (state) { + case 'attached': + this.onAttached(hasObjects); + break; + + case 'detached': + case 'failed': + // do not emit data update events as the actual current state of Objects data is unknown when we're in these channel states + this._objectsPool.clearObjectsData(false); + this._syncObjectsDataPool.clear(); + break; + } + } + + /** + * @internal + */ + async publish(objectMessages: ObjectMessage[]): Promise { + this._channel.throwIfUnpublishableState(); + + objectMessages.forEach((x) => ObjectMessage.encode(x, this._client)); + const maxMessageSize = this._client.options.maxMessageSize; + const size = objectMessages.reduce((acc, msg) => acc + msg.getMessageSize(), 0); + if (size > maxMessageSize) { + throw new this._client.ErrorInfo( + `Maximum size of object messages that can be published at once exceeded (was ${size} bytes; limit is ${maxMessageSize} bytes)`, + 40009, + 400, + ); + } + + return this._channel.sendState(objectMessages); + } + + /** + * @internal + */ + throwIfInvalidAccessApiConfiguration(): void { + this._throwIfMissingChannelMode('object_subscribe'); + this._throwIfInChannelState(['detached', 'failed']); + } + + /** + * @internal + */ + throwIfInvalidWriteApiConfiguration(): void { + this._throwIfMissingChannelMode('object_publish'); + this._throwIfInChannelState(['detached', 'failed', 'suspended']); + this._throwIfEchoMessagesDisabled(); + } + + private _startNewSync(syncId?: string, syncCursor?: string): void { + // need to discard all buffered object operation messages on new sync start + this._bufferedObjectOperations = []; + this._syncObjectsDataPool.clear(); + this._currentSyncId = syncId; + this._currentSyncCursor = syncCursor; + this._stateChange(ObjectsState.syncing, false); + } + + private _endSync(deferStateEvent: boolean): void { + this._applySync(); + // should apply buffered object operations after we applied the sync. + // can use regular object messages application logic + this._applyObjectMessages(this._bufferedObjectOperations); + + this._bufferedObjectOperations = []; + this._syncObjectsDataPool.clear(); + this._currentSyncId = undefined; + this._currentSyncCursor = undefined; + this._stateChange(ObjectsState.synced, deferStateEvent); + } + + private _parseSyncChannelSerial(syncChannelSerial: string | null | undefined): { + syncId: string | undefined; + syncCursor: string | undefined; + } { + let match: RegExpMatchArray | null; + let syncId: string | undefined = undefined; + let syncCursor: string | undefined = undefined; + if (syncChannelSerial && (match = syncChannelSerial.match(/^([\w-]+):(.*)$/))) { + syncId = match[1]; + syncCursor = match[2]; + } + + return { + syncId, + syncCursor, + }; + } + + private _applySync(): void { + if (this._syncObjectsDataPool.isEmpty()) { + return; + } + + const receivedObjectIds = new Set(); + const existingObjectUpdates: { object: LiveObject; update: LiveObjectUpdate | LiveObjectUpdateNoop }[] = []; + + for (const [objectId, entry] of this._syncObjectsDataPool.entries()) { + receivedObjectIds.add(objectId); + const existingObject = this._objectsPool.get(objectId); + + if (existingObject) { + const update = existingObject.overrideWithObjectState(entry.objectState); + // store updates to call subscription callbacks for all of them once the sync sequence is completed. + // this will ensure that clients get notified about the changes only once everything has been applied. + existingObjectUpdates.push({ object: existingObject, update }); + continue; + } + + let newObject: LiveObject; + // assign to a variable so TS doesn't complain about 'never' type in the default case + const objectType = entry.objectType; + switch (objectType) { + case 'LiveCounter': + newObject = LiveCounter.fromObjectState(this, entry.objectState); + break; + + case 'LiveMap': + newObject = LiveMap.fromObjectState(this, entry.objectState); + break; + + default: + throw new this._client.ErrorInfo(`Unknown LiveObject type: ${objectType}`, 50000, 500); + } + + this._objectsPool.set(objectId, newObject); + } + + // need to remove LiveObject instances from the ObjectsPool for which objectIds were not received during the sync sequence + this._objectsPool.deleteExtraObjectIds([...receivedObjectIds]); + + // call subscription callbacks for all updated existing objects + existingObjectUpdates.forEach(({ object, update }) => object.notifyUpdated(update)); + } + + private _applyObjectMessages(objectMessages: ObjectMessage[]): void { + for (const objectMessage of objectMessages) { + if (!objectMessage.operation) { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MAJOR, + 'Objects._applyObjectMessages()', + `object operation message is received without 'operation' field, skipping message; message id: ${objectMessage.id}, channel: ${this._channel.name}`, + ); + continue; + } + + const objectOperation = objectMessage.operation; + + switch (objectOperation.action) { + case ObjectOperationAction.MAP_CREATE: + case ObjectOperationAction.COUNTER_CREATE: + case ObjectOperationAction.MAP_SET: + case ObjectOperationAction.MAP_REMOVE: + case ObjectOperationAction.COUNTER_INC: + case ObjectOperationAction.OBJECT_DELETE: + // we can receive an op for an object id we don't have yet in the pool. instead of buffering such operations, + // we can create a zero-value object for the provided object id and apply the operation to that zero-value object. + // this also means that all objects are capable of applying the corresponding *_CREATE ops on themselves, + // since they need to be able to eventually initialize themselves from that *_CREATE op. + // so to simplify operations handling, we always try to create a zero-value object in the pool first, + // and then we can always apply the operation on the existing object in the pool. + this._objectsPool.createZeroValueObjectIfNotExists(objectOperation.objectId); + this._objectsPool.get(objectOperation.objectId)!.applyOperation(objectOperation, objectMessage); + break; + + default: + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MAJOR, + 'Objects._applyObjectMessages()', + `received unsupported action in object operation message: ${objectOperation.action}, skipping message; message id: ${objectMessage.id}, channel: ${this._channel.name}`, + ); + } + } + } + + private _throwIfMissingChannelMode(expectedMode: 'object_subscribe' | 'object_publish'): void { + // channel.modes is only populated on channel attachment, so use it only if it is set, + // otherwise as a best effort use user provided channel options + if (this._channel.modes != null && !this._channel.modes.includes(expectedMode)) { + throw new this._client.ErrorInfo(`"${expectedMode}" channel mode must be set for this operation`, 40024, 400); + } + if (!this._client.Utils.allToLowerCase(this._channel.channelOptions.modes ?? []).includes(expectedMode)) { + throw new this._client.ErrorInfo(`"${expectedMode}" channel mode must be set for this operation`, 40024, 400); + } + } + + private _stateChange(state: ObjectsState, deferEvent: boolean): void { + if (this._state === state) { + return; + } + + this._state = state; + const event = StateToEventsMap[state]; + if (!event) { + return; + } + + if (deferEvent) { + this._client.Platform.Config.nextTick(() => { + this._eventEmitterInternal.emit(event); + this._eventEmitterPublic.emit(event); + }); + } else { + this._eventEmitterInternal.emit(event); + this._eventEmitterPublic.emit(event); + } + } + + private _throwIfInChannelState(channelState: API.ChannelState[]): void { + if (channelState.includes(this._channel.state)) { + throw this._client.ErrorInfo.fromValues(this._channel.invalidStateError()); + } + } + + private _throwIfEchoMessagesDisabled(): void { + if (this._channel.client.options.echoMessages === false) { + throw new this._channel.client.ErrorInfo( + `"echoMessages" client option must be enabled for this operation`, + 40000, + 400, + ); + } + } +} diff --git a/src/plugins/objects/objectspool.ts b/src/plugins/objects/objectspool.ts new file mode 100644 index 0000000000..99a9a1f1f1 --- /dev/null +++ b/src/plugins/objects/objectspool.ts @@ -0,0 +1,119 @@ +import type BaseClient from 'common/lib/client/baseclient'; +import { DEFAULTS } from './defaults'; +import { LiveCounter } from './livecounter'; +import { LiveMap } from './livemap'; +import { LiveObject } from './liveobject'; +import { ObjectId } from './objectid'; +import { Objects } from './objects'; + +export const ROOT_OBJECT_ID = 'root'; + +/** + * @internal + */ +export class ObjectsPool { + private _client: BaseClient; + private _pool: Map; + private _gcInterval: ReturnType; + + constructor(private _objects: Objects) { + this._client = this._objects.getClient(); + this._pool = this._createInitialPool(); + this._gcInterval = setInterval(() => { + this._onGCInterval(); + }, DEFAULTS.gcInterval); + // call nodejs's Timeout.unref to not require Node.js event loop to remain active due to this interval. see https://nodejs.org/api/timers.html#timeoutunref + this._gcInterval.unref?.(); + } + + get(objectId: string): LiveObject | undefined { + return this._pool.get(objectId); + } + + /** + * Deletes objects from the pool for which object ids are not found in the provided array of ids. + */ + deleteExtraObjectIds(objectIds: string[]): void { + const poolObjectIds = [...this._pool.keys()]; + const extraObjectIds = poolObjectIds.filter((x) => !objectIds.includes(x)); + + extraObjectIds.forEach((x) => this._pool.delete(x)); + } + + set(objectId: string, liveObject: LiveObject): void { + this._pool.set(objectId, liveObject); + } + + /** + * Removes all objects but root from the pool and clears the data for root. + * Does not create a new root object, so the reference to the root object remains the same. + */ + resetToInitialPool(emitUpdateEvents: boolean): void { + // clear the pool first and keep the root object + const root = this._pool.get(ROOT_OBJECT_ID)!; + this._pool.clear(); + this._pool.set(root.getObjectId(), root); + + // clear the data, this will only clear the root object + this.clearObjectsData(emitUpdateEvents); + } + + /** + * Clears the data stored for all objects in the pool. + */ + clearObjectsData(emitUpdateEvents: boolean): void { + for (const object of this._pool.values()) { + const update = object.clearData(); + if (emitUpdateEvents) { + object.notifyUpdated(update); + } + } + } + + createZeroValueObjectIfNotExists(objectId: string): LiveObject { + const existingObject = this.get(objectId); + if (existingObject) { + return existingObject; + } + + const parsedObjectId = ObjectId.fromString(this._client, objectId); + let zeroValueObject: LiveObject; + switch (parsedObjectId.type) { + case 'map': { + zeroValueObject = LiveMap.zeroValue(this._objects, objectId); + break; + } + + case 'counter': + zeroValueObject = LiveCounter.zeroValue(this._objects, objectId); + break; + } + + this.set(objectId, zeroValueObject); + return zeroValueObject; + } + + private _createInitialPool(): Map { + const pool = new Map(); + const root = LiveMap.zeroValue(this._objects, ROOT_OBJECT_ID); + pool.set(root.getObjectId(), root); + return pool; + } + + private _onGCInterval(): void { + const toDelete: string[] = []; + for (const [objectId, obj] of this._pool.entries()) { + // tombstoned objects should be removed from the pool if they have been tombstoned for longer than grace period. + // by removing them from the local pool, Objects plugin no longer keeps a reference to those objects, allowing JS's + // Garbage Collection to eventually free the memory for those objects, provided the user no longer references them either. + if (obj.isTombstoned() && Date.now() - obj.tombstonedAt()! >= DEFAULTS.gcGracePeriod) { + toDelete.push(objectId); + continue; + } + + obj.onGCInterval(); + } + + toDelete.forEach((x) => this._pool.delete(x)); + } +} diff --git a/src/plugins/objects/syncobjectsdatapool.ts b/src/plugins/objects/syncobjectsdatapool.ts new file mode 100644 index 0000000000..e411b502a5 --- /dev/null +++ b/src/plugins/objects/syncobjectsdatapool.ts @@ -0,0 +1,98 @@ +import type BaseClient from 'common/lib/client/baseclient'; +import type RealtimeChannel from 'common/lib/client/realtimechannel'; +import { ObjectMessage, ObjectState } from './objectmessage'; +import { Objects } from './objects'; + +export interface LiveObjectDataEntry { + objectState: ObjectState; + objectType: 'LiveMap' | 'LiveCounter'; +} + +export interface LiveCounterDataEntry extends LiveObjectDataEntry { + objectType: 'LiveCounter'; +} + +export interface LiveMapDataEntry extends LiveObjectDataEntry { + objectType: 'LiveMap'; +} + +export type AnyDataEntry = LiveCounterDataEntry | LiveMapDataEntry; + +// TODO: investigate if this class is still needed after changes with createOp. objects are now initialized from the stateObject and this class does minimal processing +/** + * @internal + */ +export class SyncObjectsDataPool { + private _client: BaseClient; + private _channel: RealtimeChannel; + private _pool: Map; + + constructor(private _objects: Objects) { + this._client = this._objects.getClient(); + this._channel = this._objects.getChannel(); + this._pool = new Map(); + } + + entries() { + return this._pool.entries(); + } + + size(): number { + return this._pool.size; + } + + isEmpty(): boolean { + return this._pool.size === 0; + } + + clear(): void { + this._pool.clear(); + } + + applyObjectSyncMessages(objectMessages: ObjectMessage[]): void { + for (const objectMessage of objectMessages) { + if (!objectMessage.object) { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MAJOR, + 'SyncObjectsDataPool.applyObjectSyncMessages()', + `object message is received during OBJECT_SYNC without 'object' field, skipping message; message id: ${objectMessage.id}, channel: ${this._channel.name}`, + ); + continue; + } + + const objectState = objectMessage.object; + + if (objectState.counter) { + this._pool.set(objectState.objectId, this._createLiveCounterDataEntry(objectState)); + } else if (objectState.map) { + this._pool.set(objectState.objectId, this._createLiveMapDataEntry(objectState)); + } else { + this._client.Logger.logAction( + this._client.logger, + this._client.Logger.LOG_MAJOR, + 'SyncObjectsDataPool.applyObjectSyncMessages()', + `received unsupported object state message during OBJECT_SYNC, expected 'counter' or 'map' to be present, skipping message; message id: ${objectMessage.id}, channel: ${this._channel.name}`, + ); + } + } + } + + private _createLiveCounterDataEntry(objectState: ObjectState): LiveCounterDataEntry { + const newEntry: LiveCounterDataEntry = { + objectState, + objectType: 'LiveCounter', + }; + + return newEntry; + } + + private _createLiveMapDataEntry(objectState: ObjectState): LiveMapDataEntry { + const newEntry: LiveMapDataEntry = { + objectState, + objectType: 'LiveMap', + }; + + return newEntry; + } +} diff --git a/test/browser/modular.test.js b/test/browser/modular.test.js index b949db16be..94a0460f9e 100644 --- a/test/browser/modular.test.js +++ b/test/browser/modular.test.js @@ -515,11 +515,7 @@ function registerAblyModularTests(Helper) { }, }); - await ( - clientClassConfig.isRealtime - ? (op, realtime) => helper.monitorConnectionThenCloseAndFinishAsync(op, realtime) - : async (op) => await op() - )(async () => { + const action = async () => { const txChannel = txClient.channels.get('channel', encryptionChannelOptions); await txChannel.publish(txMessage); @@ -531,7 +527,13 @@ function registerAblyModularTests(Helper) { // Verify that the message was correctly encrypted const rxMessageDecrypted = await decodeEncryptedMessage(rxMessage, encryptionChannelOptions); helper.testMessageEquality(rxMessageDecrypted, txMessage); - }, txClient); + }; + + if (clientClassConfig.isRealtime) { + await helper.monitorConnectionThenCloseAndFinishAsync(action, txClient); + } else { + await action(); + } }, rxClient); } diff --git a/test/common/globals/named_dependencies.js b/test/common/globals/named_dependencies.js index e06d925987..5e9367ff45 100644 --- a/test/common/globals/named_dependencies.js +++ b/test/common/globals/named_dependencies.js @@ -11,6 +11,10 @@ define(function () { browser: 'build/push', node: 'build/push', }, + objects: { + browser: 'build/objects', + node: 'build/objects', + }, // test modules globals: { browser: 'test/common/globals/environment', node: 'test/common/globals/environment' }, @@ -18,9 +22,14 @@ define(function () { async: { browser: 'node_modules/async/lib/async' }, chai: { browser: 'node_modules/chai/chai', node: 'node_modules/chai/chai' }, ulid: { browser: 'node_modules/ulid/dist/index.umd', node: 'node_modules/ulid/dist/index.umd' }, + dequal: { browser: 'node_modules/dequal/dist/index.min', node: 'node_modules/dequal/dist/index' }, private_api_recorder: { browser: 'test/common/modules/private_api_recorder', node: 'test/common/modules/private_api_recorder', }, + objects_helper: { + browser: 'test/common/modules/objects_helper', + node: 'test/common/modules/objects_helper', + }, }); }); diff --git a/test/common/modules/objects_helper.js b/test/common/modules/objects_helper.js new file mode 100644 index 0000000000..ff8e48b96e --- /dev/null +++ b/test/common/modules/objects_helper.js @@ -0,0 +1,419 @@ +'use strict'; + +/** + * Helper class to create pre-determined objects tree on channels and create object messages. + */ +define(['ably', 'shared_helper', 'objects'], function (Ably, Helper, ObjectsPlugin) { + const createPM = Ably.makeProtocolMessageFromDeserialized({ ObjectsPlugin }); + + const ACTIONS = { + MAP_CREATE: 0, + MAP_SET: 1, + MAP_REMOVE: 2, + COUNTER_CREATE: 3, + COUNTER_INC: 4, + OBJECT_DELETE: 5, + }; + const ACTION_STRINGS = { + MAP_CREATE: 'MAP_CREATE', + MAP_SET: 'MAP_SET', + MAP_REMOVE: 'MAP_REMOVE', + COUNTER_CREATE: 'COUNTER_CREATE', + COUNTER_INC: 'COUNTER_INC', + OBJECT_DELETE: 'OBJECT_DELETE', + }; + + function nonce() { + return Helper.randomString(); + } + + class ObjectsHelper { + constructor(helper) { + this._helper = helper; + this._rest = helper.AblyRest({ useBinaryProtocol: false }); + } + + static ACTIONS = ACTIONS; + + static fixtureRootKeys() { + return ['emptyCounter', 'initialValueCounter', 'referencedCounter', 'emptyMap', 'referencedMap', 'valuesMap']; + } + + /** + * Sends Objects REST API requests to create objects tree on a provided channel: + * + * root "emptyMap" -> Map#1 {} -- empty map + * root "referencedMap" -> Map#2 { "counterKey": } + * root "valuesMap" -> Map#3 { "stringKey": "stringValue", "emptyStringKey": "", "bytesKey": , "emptyBytesKey": , "numberKey": 1, "zeroKey": 0, "trueKey": true, "falseKey": false, "mapKey": } + * root "emptyCounter" -> Counter#1 -- no initial value counter, should be 0 + * root "initialValueCounter" -> Counter#2 count=10 + * root "referencedCounter" -> Counter#3 count=20 + */ + async initForChannel(channelName) { + const emptyCounter = await this.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'emptyCounter', + createOp: this.counterCreateRestOp(), + }); + const initialValueCounter = await this.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'initialValueCounter', + createOp: this.counterCreateRestOp({ number: 10 }), + }); + const referencedCounter = await this.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'referencedCounter', + createOp: this.counterCreateRestOp({ number: 20 }), + }); + + const emptyMap = await this.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'emptyMap', + createOp: this.mapCreateRestOp(), + }); + const referencedMap = await this.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'referencedMap', + createOp: this.mapCreateRestOp({ data: { counterKey: { objectId: referencedCounter.objectId } } }), + }); + const valuesMap = await this.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'valuesMap', + createOp: this.mapCreateRestOp({ + data: { + stringKey: { string: 'stringValue' }, + emptyStringKey: { string: '' }, + bytesKey: { bytes: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9' }, + emptyBytesKey: { bytes: '' }, + numberKey: { number: 1 }, + zeroKey: { number: 0 }, + trueKey: { boolean: true }, + falseKey: { boolean: false }, + mapKey: { objectId: referencedMap.objectId }, + }, + }), + }); + } + + // #region Wire Object Messages + + mapCreateOp(opts) { + const { objectId, entries } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.MAP_CREATE, + nonce: nonce(), + objectId, + map: { + semantics: 0, + }, + }, + }; + + if (entries) { + op.operation.map = { + ...op.operation.map, + entries, + }; + } + + return op; + } + + mapSetOp(opts) { + const { objectId, key, data } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.MAP_SET, + objectId, + mapOp: { + key, + data, + }, + }, + }; + + return op; + } + + mapRemoveOp(opts) { + const { objectId, key } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.MAP_REMOVE, + objectId, + mapOp: { + key, + }, + }, + }; + + return op; + } + + counterCreateOp(opts) { + const { objectId, count } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.COUNTER_CREATE, + nonce: nonce(), + objectId, + }, + }; + + if (count != null) { + op.operation.counter = { count }; + } + + return op; + } + + counterIncOp(opts) { + const { objectId, amount } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.COUNTER_INC, + objectId, + counterOp: { + amount, + }, + }, + }; + + return op; + } + + objectDeleteOp(opts) { + const { objectId } = opts ?? {}; + const op = { + operation: { + action: ACTIONS.OBJECT_DELETE, + objectId, + }, + }; + + return op; + } + + mapObject(opts) { + const { objectId, siteTimeserials, initialEntries, materialisedEntries, tombstone } = opts; + const obj = { + object: { + objectId, + siteTimeserials, + tombstone: tombstone === true, + map: { + semantics: 0, + entries: materialisedEntries, + }, + }, + }; + + if (initialEntries) { + obj.object.createOp = this.mapCreateOp({ objectId, entries: initialEntries }).operation; + } + + return obj; + } + + counterObject(opts) { + const { objectId, siteTimeserials, initialCount, materialisedCount, tombstone } = opts; + const obj = { + object: { + objectId, + siteTimeserials, + tombstone: tombstone === true, + counter: { + count: materialisedCount, + }, + }, + }; + + if (initialCount != null) { + obj.object.createOp = this.counterCreateOp({ objectId, count: initialCount }).operation; + } + + return obj; + } + + objectOperationMessage(opts) { + const { channelName, serial, siteCode, state } = opts; + + state?.forEach((objectMessage, i) => { + objectMessage.serial = serial; + objectMessage.siteCode = siteCode; + }); + + return { + action: 19, // OBJECT + channel: channelName, + channelSerial: serial, + state: state ?? [], + }; + } + + objectStateMessage(opts) { + const { channelName, syncSerial, state } = opts; + + return { + action: 20, // OBJECT_SYNC + channel: channelName, + channelSerial: syncSerial, + state: state ?? [], + }; + } + + async processObjectOperationMessageOnChannel(opts) { + const { channel, ...rest } = opts; + + this._helper.recordPrivateApi('call.channel.processMessage'); + this._helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + await channel.processMessage( + createPM( + this.objectOperationMessage({ + ...rest, + channelName: channel.name, + }), + ), + ); + } + + async processObjectStateMessageOnChannel(opts) { + const { channel, ...rest } = opts; + + this._helper.recordPrivateApi('call.channel.processMessage'); + this._helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + await channel.processMessage( + createPM( + this.objectStateMessage({ + ...rest, + channelName: channel.name, + }), + ), + ); + } + + // #endregion + + // #region REST API Operations + + async createAndSetOnMap(channelName, opts) { + const { mapObjectId, key, createOp } = opts; + + const createResult = await this.operationRequest(channelName, createOp); + const objectId = createResult.objectId; + await this.operationRequest(channelName, this.mapSetRestOp({ objectId: mapObjectId, key, value: { objectId } })); + + return createResult; + } + + mapCreateRestOp(opts) { + const { objectId, nonce, data } = opts ?? {}; + const opBody = { + operation: ACTION_STRINGS.MAP_CREATE, + }; + + if (data) { + opBody.data = data; + } + + if (objectId != null) { + opBody.objectId = objectId; + opBody.nonce = nonce; + } + + return opBody; + } + + mapSetRestOp(opts) { + const { objectId, key, value } = opts ?? {}; + const opBody = { + operation: ACTION_STRINGS.MAP_SET, + objectId, + data: { + key, + value, + }, + }; + + return opBody; + } + + mapRemoveRestOp(opts) { + const { objectId, key } = opts ?? {}; + const opBody = { + operation: ACTION_STRINGS.MAP_REMOVE, + objectId, + data: { + key, + }, + }; + + return opBody; + } + + counterCreateRestOp(opts) { + const { objectId, nonce, number } = opts ?? {}; + const opBody = { + operation: ACTION_STRINGS.COUNTER_CREATE, + }; + + if (number != null) { + opBody.data = { number }; + } + + if (objectId != null) { + opBody.objectId = objectId; + opBody.nonce = nonce; + } + + return opBody; + } + + counterIncRestOp(opts) { + const { objectId, number } = opts ?? {}; + const opBody = { + operation: ACTION_STRINGS.COUNTER_INC, + objectId, + data: { number }, + }; + + return opBody; + } + + async operationRequest(channelName, opBody) { + if (Array.isArray(opBody)) { + throw new Error(`Only single object operation requests are supported`); + } + + const method = 'post'; + const path = `/channels/${channelName}/objects`; + + const response = await this._rest.request(method, path, 3, null, opBody, null); + + if (response.success) { + // only one operation in the request, so need only the first item. + const result = response.items[0]; + // extract objectId if present + result.objectId = result.objectIds?.[0]; + return result; + } + + throw new Error( + `${method}: ${path} FAILED; http code = ${response.statusCode}, error code = ${response.errorCode}, message = ${response.errorMessage}; operation = ${JSON.stringify(opBody)}`, + ); + } + + // #endregion + + fakeMapObjectId() { + return `map:${Helper.randomString()}@${Date.now()}`; + } + + fakeCounterObjectId() { + return `counter:${Helper.randomString()}@${Date.now()}`; + } + } + + return (module.exports = ObjectsHelper); +}); diff --git a/test/common/modules/private_api_recorder.js b/test/common/modules/private_api_recorder.js index 57cc6c55d3..cecf6670b8 100644 --- a/test/common/modules/private_api_recorder.js +++ b/test/common/modules/private_api_recorder.js @@ -16,13 +16,21 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'call.Defaults.getPort', 'call.Defaults.normaliseOptions', 'call.EventEmitter.emit', + 'call.LiveObject.getObjectId', + 'call.LiveObject.isTombstoned', + 'call.Objects._objectsPool._onGCInterval', + 'call.Objects._objectsPool.get', 'call.Message.decode', 'call.Message.encode', + 'call.ObjectMessage.encode', + 'call.ObjectMessage.fromValues', + 'call.ObjectMessage.getMessageSize', 'call.Platform.Config.push.storage.clear', 'call.Platform.nextTick', 'call.PresenceMessage.fromValues', 'call.ProtocolMessage.setFlag', 'call.Utils.copy', + 'call.Utils.dataSizeBytes', 'call.Utils.getRetryTime', 'call.Utils.inspectError', 'call.Utils.keysArray', @@ -48,7 +56,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'call.msgpack.encode', 'call.presence._myMembers.put', 'call.presence.waitSync', - 'call.protocolMessageFromDeserialized', + 'call.makeProtocolMessageFromDeserialized', 'call.realtime.baseUri', 'call.rest.baseUri', 'call.rest.http.do', @@ -71,8 +79,10 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'pass.clientOption.webSocketSlowTimeout', 'pass.clientOption.wsConnectivityCheckUrl', // actually ably-js public API (i.e. it’s in the TypeScript typings) but no other SDK has it. At the same time it's not entirely clear if websocket connectivity check should be considered an ably-js-specific functionality (as for other params above), so for the time being we consider it as private API 'read.Defaults.version', + 'read.LiveMap._dataRef.data', 'read.EventEmitter.events', 'read.Platform.Config.push', + 'read.ProtocolMessage.channelSerial', 'read.Realtime._transports', 'read.auth.authOptions.authUrl', 'read.auth.key', @@ -80,6 +90,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'read.auth.tokenParams.version', 'read.channel.channelOptions', 'read.channel.channelOptions.cipher', + 'read.channel.properties.channelSerial', // This should be public API, but channel.properties is not currently exposed. Remove it from the list when https://github.com/ably/ably-js/issues/2018 is done 'read.connectionManager.activeProtocol', 'read.connectionManager.activeProtocol.transport', 'read.connectionManager.baseTransport', @@ -101,6 +112,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'read.realtime.options.maxMessageSize', 'read.realtime.options.realtimeHost', 'read.realtime.options.token', + 'read.realtime.options.useBinaryProtocol', 'read.rest._currentFallback', 'read.rest._currentFallback.host', 'read.rest._currentFallback.validUntil', @@ -111,6 +123,8 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'read.transport.params.mode', 'read.transport.recvRequest.recvUri', 'read.transport.uri', + 'replace.Objects._objectsPool._onGCInterval', + 'replace.Objects.publish', 'replace.channel.attachImpl', 'replace.channel.processMessage', 'replace.channel.sendMessage', @@ -127,11 +141,14 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'serialize.recoveryKey', 'write.Defaults.ENVIRONMENT', 'write.Defaults.wsConnectivityCheckUrl', + 'write.Objects._DEFAULTS.gcGracePeriod', + 'write.Objects._DEFAULTS.gcInterval', 'write.Platform.Config.push', // This implies using a mock implementation of the internal IPlatformPushConfig interface. Our mock (in push_channel_transport.js) then interacts with internal objects and private APIs of public objects to implement this interface; I haven’t added annotations for that private API usage, since there wasn’t an easy way to pass test context information into the mock. I think that for now we can just say that if we wanted to get rid of this private API usage, then we’d need to remove this mock entirely. 'write.auth.authOptions.requestHeaders', 'write.auth.key', 'write.auth.tokenDetails.token', 'write.channel._lastPayload', + 'write.channel.channelOptions.modes', 'write.channel.state', 'write.connectionManager.connectionDetails.maxMessageSize', 'write.connectionManager.connectionId', @@ -139,6 +156,7 @@ define(['test/support/output_directory_paths'], function (outputDirectoryPaths) 'write.connectionManager.lastActivity', 'write.connectionManager.msgSerial', 'write.connectionManager.wsHosts', + 'write.realtime.options.echoMessages', 'write.realtime.options.realtimeHost', 'write.realtime.options.wsConnectivityCheckUrl', 'write.realtime.options.timeouts.realtimeRequestTimeout', diff --git a/test/common/modules/shared_helper.js b/test/common/modules/shared_helper.js index f754b173fb..9b1a1b15fd 100644 --- a/test/common/modules/shared_helper.js +++ b/test/common/modules/shared_helper.js @@ -334,48 +334,63 @@ define([ ); } - /* testFn is assumed to be a function of realtimeOptions that returns a mocha test */ - static testOnAllTransports(thisInDescribe, name, testFn, skip) { - const helper = this.forTestDefinition(thisInDescribe, name).addingHelperFunction('testOnAllTransports'); - var itFn = skip ? it.skip : it; + /* testFn is assumed to be a function of realtimeOptions and channelName that returns a mocha test */ + static testOnAllTransportsAndProtocols(thisInDescribe, testName, testFn, skip, only) { + const helper = SharedHelper.forTestDefinition(thisInDescribe, testName).addingHelperFunction( + 'testOnAllTransportsAndProtocols', + ); + const itFn = skip ? it.skip : only ? it.only : it; - function createTest(options) { + function createTest(options, channelName) { return function (done) { - this.test.helper = this.test.helper.withParameterisedTestTitle(name); - return testFn(options).apply(this, [done]); + this.test.helper = this.test.helper.withParameterisedTestTitle(testName); + // we want to support both callback-based and async test functions here. + // for this we check the return type of the test function to see if it is a Promise. + // if it is, then the test function provided is an async one, and won't call done function on its own. + // instead we attach own .then and .catch callbacks for the promise here and call done when needed + const testFnReturn = testFn(options, channelName).apply(this, [done]); + if (testFnReturn instanceof Promise) { + testFnReturn.then(done).catch(done); + } else { + return testFnReturn; + } }; } - let transports = helper.availableTransports; + const transports = helper.availableTransports; transports.forEach(function (transport) { itFn( - name + '_with_' + transport + '_binary_transport', - createTest({ transports: [transport], useBinaryProtocol: true }), + testName + ' with ' + transport + ' binary protocol', + createTest({ transports: [transport], useBinaryProtocol: true }, `${testName} ${transport} binary`), ); itFn( - name + '_with_' + transport + '_text_transport', - createTest({ transports: [transport], useBinaryProtocol: false }), + testName + ' with ' + transport + ' text protocol', + createTest({ transports: [transport], useBinaryProtocol: false }, `${testName} ${transport} text`), ); }); /* Plus one for no transport specified (ie use websocket/base mechanism if * present). (we explicitly specify all transports since node only does * websocket+nodecomet if comet is explicitly requested) * */ - itFn(name + '_with_binary_transport', createTest({ transports, useBinaryProtocol: true })); - itFn(name + '_with_text_transport', createTest({ transports, useBinaryProtocol: false })); + itFn( + testName + ' with binary protocol', + createTest({ transports, useBinaryProtocol: true }, `${testName} binary`), + ); + itFn(testName + ' with text protocol', createTest({ transports, useBinaryProtocol: false }, `${testName} text`)); } - static restTestOnJsonMsgpack(name, testFn, skip) { - var itFn = skip ? it.skip : it; - itFn(name + ' with binary protocol', async function () { - const helper = this.test.helper.withParameterisedTestTitle(name); + static testOnJsonMsgpack(testName, testFn, skip, only) { + const itFn = skip ? it.skip : only ? it.only : it; - await testFn(new clientModule.AblyRest(helper, { useBinaryProtocol: true }), name + '_binary', helper); + itFn(testName + ' with binary protocol', async function () { + const helper = this.test.helper.withParameterisedTestTitle(testName); + const channelName = testName + ' binary'; + await testFn({ useBinaryProtocol: true }, channelName, helper); }); - itFn(name + ' with text protocol', async function () { - const helper = this.test.helper.withParameterisedTestTitle(name); - - await testFn(new clientModule.AblyRest(helper, { useBinaryProtocol: false }), name + '_text', helper); + itFn(testName + ' with text protocol', async function () { + const helper = this.test.helper.withParameterisedTestTitle(testName); + const channelName = testName + ' text'; + await testFn({ useBinaryProtocol: false }, channelName, helper); }); } @@ -508,12 +523,20 @@ define([ } } - SharedHelper.testOnAllTransports.skip = function (thisInDescribe, name, testFn) { - SharedHelper.testOnAllTransports(thisInDescribe, name, testFn, true); + SharedHelper.testOnAllTransportsAndProtocols.skip = function (thisInDescribe, testName, testFn) { + SharedHelper.testOnAllTransportsAndProtocols(thisInDescribe, testName, testFn, true); + }; + + SharedHelper.testOnAllTransportsAndProtocols.only = function (thisInDescribe, testName, testFn) { + SharedHelper.testOnAllTransportsAndProtocols(thisInDescribe, testName, testFn, false, true); + }; + + SharedHelper.testOnJsonMsgpack.skip = function (testName, testFn) { + SharedHelper.testOnJsonMsgpack(testName, testFn, true); }; - SharedHelper.restTestOnJsonMsgpack.skip = function (name, testFn) { - SharedHelper.restTestOnJsonMsgpack(name, testFn, true); + SharedHelper.testOnJsonMsgpack.only = function (testName, testFn) { + SharedHelper.testOnJsonMsgpack(testName, testFn, false, true); }; return (module.exports = SharedHelper); diff --git a/test/package/browser/template/README.md b/test/package/browser/template/README.md index 8bf44061be..cc5cbc54c2 100644 --- a/test/package/browser/template/README.md +++ b/test/package/browser/template/README.md @@ -8,6 +8,7 @@ This directory is intended to be used for testing the following aspects of the a It contains three files, each of which import ably-js in different manners, and provide a way to briefly exercise its functionality: - `src/index-default.ts` imports the default ably-js package (`import { Realtime } from 'ably'`). +- `src/index-objects.ts` imports the Objects ably-js plugin (`import Objects from 'ably/objects'`). - `src/index-modular.ts` imports the tree-shakable ably-js package (`import { BaseRealtime, WebSocketTransport, FetchRequest } from 'ably/modular'`). - `src/ReactApp.tsx` imports React hooks from the ably-js package (`import { useChannel } from 'ably/react'`). @@ -25,6 +26,7 @@ This directory exposes three package scripts that are to be used for testing: - `build`: Uses esbuild to create: 1. a bundle containing `src/index-default.ts` and ably-js; - 2. a bundle containing `src/index-modular.ts` and ably-js. + 2. a bundle containing `src/index-objects.ts` and ably-js. + 3. a bundle containing `src/index-modular.ts` and ably-js. - `test`: Using the bundles created by `build` and playwright components setup, tests that the code that exercises ably-js’s functionality is working correctly in a browser. - `typecheck`: Type-checks the code that imports ably-js. diff --git a/test/package/browser/template/package.json b/test/package/browser/template/package.json index a05aa04977..574da1a7b3 100644 --- a/test/package/browser/template/package.json +++ b/test/package/browser/template/package.json @@ -4,7 +4,7 @@ "description": "", "main": "index.js", "scripts": { - "build": "esbuild --bundle src/index-default.ts --outdir=dist && esbuild --bundle src/index-modular.ts --outdir=dist", + "build": "esbuild --bundle src/index-default.ts --outdir=dist && esbuild --bundle src/index-objects.ts --outdir=dist && esbuild --bundle src/index-modular.ts --outdir=dist", "typecheck": "tsc --project src -noEmit", "test-support:server": "ts-node server/server.ts", "test": "npm run test:lib && npm run test:hooks", diff --git a/test/package/browser/template/server/resources/index-objects.html b/test/package/browser/template/server/resources/index-objects.html new file mode 100644 index 0000000000..44d594e83c --- /dev/null +++ b/test/package/browser/template/server/resources/index-objects.html @@ -0,0 +1,11 @@ + + + + + Ably NPM package test (Objects plugin export) + + + + + + diff --git a/test/package/browser/template/server/server.ts b/test/package/browser/template/server/server.ts index 2409fc30c5..0cac0b7f18 100644 --- a/test/package/browser/template/server/server.ts +++ b/test/package/browser/template/server/server.ts @@ -5,7 +5,7 @@ async function startWebServer(listenPort: number) { const server = express(); server.get('/', (req, res) => res.send('OK')); server.use(express.static(path.join(__dirname, '/resources'))); - for (const filename of ['index-default.js', 'index-modular.js']) { + for (const filename of ['index-default.js', 'index-objects.js', 'index-modular.js']) { server.use(`/${filename}`, express.static(path.join(__dirname, '..', 'dist', filename))); } diff --git a/test/package/browser/template/src/index-default.ts b/test/package/browser/template/src/index-default.ts index cda822d21b..862d0bd9bc 100644 --- a/test/package/browser/template/src/index-default.ts +++ b/test/package/browser/template/src/index-default.ts @@ -1,6 +1,12 @@ import * as Ably from 'ably'; import { createSandboxAblyAPIKey } from './sandbox'; +// Fix for "type 'typeof globalThis' has no index signature" error: +// https://stackoverflow.com/questions/68481686/type-typeof-globalthis-has-no-index-signature +declare module globalThis { + var testAblyPackage: () => Promise; +} + // This function exists to check that we can refer to the types exported by Ably. async function attachChannel(channel: Ably.RealtimeChannel) { await channel.attach(); diff --git a/test/package/browser/template/src/index-modular.ts b/test/package/browser/template/src/index-modular.ts index a9186f56dd..46b06e4617 100644 --- a/test/package/browser/template/src/index-modular.ts +++ b/test/package/browser/template/src/index-modular.ts @@ -2,6 +2,12 @@ import { BaseRealtime, WebSocketTransport, FetchRequest, generateRandomKey } fro import { InboundMessage, RealtimeChannel } from 'ably'; import { createSandboxAblyAPIKey } from './sandbox'; +// Fix for "type 'typeof globalThis' has no index signature" error: +// https://stackoverflow.com/questions/68481686/type-typeof-globalthis-has-no-index-signature +declare module globalThis { + var testAblyPackage: () => Promise; +} + // This function exists to check that we can refer to the types exported by Ably. async function attachChannel(channel: RealtimeChannel) { await channel.attach(); diff --git a/test/package/browser/template/src/index-objects.ts b/test/package/browser/template/src/index-objects.ts new file mode 100644 index 0000000000..ada25ba889 --- /dev/null +++ b/test/package/browser/template/src/index-objects.ts @@ -0,0 +1,95 @@ +import * as Ably from 'ably'; +import Objects from 'ably/objects'; +import { createSandboxAblyAPIKey } from './sandbox'; + +// Fix for "type 'typeof globalThis' has no index signature" error: +// https://stackoverflow.com/questions/68481686/type-typeof-globalthis-has-no-index-signature +declare module globalThis { + var testAblyPackage: () => Promise; +} + +type CustomRoot = { + numberKey: number; + stringKey: string; + booleanKey: boolean; + couldBeUndefined?: string; + mapKey: Ably.LiveMap<{ + foo: 'bar'; + nestedMap?: Ably.LiveMap<{ + baz: 'qux'; + }>; + }>; + counterKey: Ably.LiveCounter; +}; + +declare global { + export interface AblyObjectsTypes { + root: CustomRoot; + } +} + +type ExplicitRootType = { + someOtherKey: string; +}; + +globalThis.testAblyPackage = async function () { + const key = await createSandboxAblyAPIKey(); + + const realtime = new Ably.Realtime({ key, environment: 'sandbox', plugins: { Objects } }); + + const channel = realtime.channels.get('channel', { modes: ['OBJECT_SUBSCRIBE', 'OBJECT_PUBLISH'] }); + // check Objects can be accessed + const objects = channel.objects; + await channel.attach(); + // expect root to be a LiveMap instance with Objects types defined via the global AblyObjectsTypes interface + // also checks that we can refer to the Objects types exported from 'ably' by referencing a LiveMap interface + const root: Ably.LiveMap = await objects.getRoot(); + + // check root has expected LiveMap TypeScript type methods + const size: number = root.size(); + + // check custom user provided typings via AblyObjectsTypes are working: + // any LiveMap.get() call can return undefined, as the LiveMap itself can be tombstoned (has empty state), + // or referenced object is tombstoned. + // keys on a root: + const aNumber: number | undefined = root.get('numberKey'); + const aString: string | undefined = root.get('stringKey'); + const aBoolean: boolean | undefined = root.get('booleanKey'); + const userProvidedUndefined: string | undefined = root.get('couldBeUndefined'); + // objects on a root: + const counter: Ably.LiveCounter | undefined = root.get('counterKey'); + const map: AblyObjectsTypes['root']['mapKey'] | undefined = root.get('mapKey'); + // check string literal types works + // need to use nullish coalescing as we didn't actually create any data on the root, + // so the next calls would fail. we only need to check that TypeScript types work + const foo: 'bar' = map?.get('foo')!; + const baz: 'qux' = map?.get('nestedMap')?.get('baz')!; + + // check LiveMap subscription callback has correct TypeScript types + const { unsubscribe } = root.subscribe(({ update }) => { + // check update object infers keys from map type + const typedKeyOnMap = update.stringKey; + switch (typedKeyOnMap) { + case 'removed': + case 'updated': + case undefined: + break; + default: + // check all possible types are exhausted + const shouldExhaustAllTypes: never = typedKeyOnMap; + } + }); + unsubscribe(); + + // check LiveCounter type also behaves as expected + // same deal with nullish coalescing + const value: number = counter?.value()!; + const counterSubscribeResponse = counter?.subscribe(({ update }) => { + const shouldBeANumber: number = update.amount; + }); + counterSubscribeResponse?.unsubscribe(); + + // check can provide custom types for the getRoot method, ignoring global AblyObjectsTypes interface + const explicitRoot: Ably.LiveMap = await objects.getRoot(); + const someOtherKey: string | undefined = explicitRoot.get('someOtherKey'); +}; diff --git a/test/package/browser/template/src/sandbox.ts b/test/package/browser/template/src/sandbox.ts index 100ab22f00..54c5f2e64a 100644 --- a/test/package/browser/template/src/sandbox.ts +++ b/test/package/browser/template/src/sandbox.ts @@ -1,10 +1,14 @@ import testAppSetup from '../../../../common/ably-common/test-resources/test-app-setup.json'; -export async function createSandboxAblyAPIKey() { +export async function createSandboxAblyAPIKey(withOptions?: object) { + const postData = { + ...testAppSetup.post_apps, + ...(withOptions ?? {}), + }; const response = await fetch('https://sandbox-rest.ably.io/apps', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(testAppSetup.post_apps), + body: JSON.stringify(postData), }); if (!response.ok) { diff --git a/test/package/browser/template/src/tsconfig.json b/test/package/browser/template/src/tsconfig.json index e280baa6de..1640a1da14 100644 --- a/test/package/browser/template/src/tsconfig.json +++ b/test/package/browser/template/src/tsconfig.json @@ -1,6 +1,7 @@ { "include": ["**/*.ts", "**/*.tsx"], "compilerOptions": { + "strict": true, "resolveJsonModule": true, "esModuleInterop": true, "module": "esnext", diff --git a/test/package/browser/template/test/lib/package.test.ts b/test/package/browser/template/test/lib/package.test.ts index fc73006f3d..6c903d7f4a 100644 --- a/test/package/browser/template/test/lib/package.test.ts +++ b/test/package/browser/template/test/lib/package.test.ts @@ -3,6 +3,7 @@ import { test, expect } from '@playwright/test'; test.describe('NPM package', () => { for (const scenario of [ { name: 'default export', path: '/index-default.html' }, + { name: 'Objects plugin export', path: '/index-objects.html' }, { name: 'modular export', path: '/index-modular.html' }, ]) { test.describe(scenario.name, () => { diff --git a/test/realtime/auth.test.js b/test/realtime/auth.test.js index 3cdd43dcbc..6af1a7a319 100644 --- a/test/realtime/auth.test.js +++ b/test/realtime/auth.test.js @@ -774,7 +774,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RSA4b1 * @specpartial RSA4b - token expired */ - Helper.testOnAllTransports(this, 'auth_token_expires', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'auth_token_expires', function (realtimeOpts) { return function (done) { var helper = this.test.helper, clientRealtime, @@ -879,7 +879,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @specpartial RTN15a - attempt to reconnect and restore the connection state on token expire * @specpartial RSA10e - obtain new token from authcallback when previous expires */ - Helper.testOnAllTransports(this, 'auth_tokenDetails_expiry_with_authcallback', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'auth_tokenDetails_expiry_with_authcallback', function (realtimeOpts) { return function (done) { var helper = this.test.helper, realtime, @@ -928,7 +928,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @specpartial RTN15a - attempt to reconnect and restore the connection state on token expire * @specpartial RSA10e - obtain new token from authcallback when previous expires */ - Helper.testOnAllTransports(this, 'auth_token_string_expiry_with_authcallback', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'auth_token_string_expiry_with_authcallback', function (realtimeOpts) { return function (done) { var helper = this.test.helper, realtime, @@ -975,7 +975,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RSA4a * @spec RSA4a2 */ - Helper.testOnAllTransports(this, 'auth_token_string_expiry_with_token', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'auth_token_string_expiry_with_token', function (realtimeOpts) { return function (done) { var helper = this.test.helper, realtime, @@ -1023,7 +1023,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RSA4a * @spec RSA4a2 */ - Helper.testOnAllTransports(this, 'auth_expired_token_string', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'auth_expired_token_string', function (realtimeOpts) { return function (done) { var helper = this.test.helper, realtime, @@ -1073,7 +1073,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTC8 * @specskip */ - Helper.testOnAllTransports.skip(this, 'reauth_authCallback', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols.skip(this, 'reauth_authCallback', function (realtimeOpts) { return function (done) { var helper = this.test.helper, realtime, @@ -1586,7 +1586,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); /** @nospec */ - Helper.testOnAllTransports(this, 'authorize_immediately_after_init', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'authorize_immediately_after_init', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var realtime = helper.AblyRealtime({ diff --git a/test/realtime/channel.test.js b/test/realtime/channel.test.js index cf6af3f722..2f3280e200 100644 --- a/test/realtime/channel.test.js +++ b/test/realtime/channel.test.js @@ -4,7 +4,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async var exports = {}; var _exports = {}; var expect = chai.expect; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); function checkCanSubscribe(channel, testChannel) { return function (callback) { @@ -170,7 +170,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTS3c * @spec RTL16 */ - Helper.testOnAllTransports(this, 'channelinit0', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelinit0', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -208,7 +208,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @spec RTL4 */ - Helper.testOnAllTransports(this, 'channelattach0', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattach0', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -235,7 +235,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @spec RTL4 */ - Helper.testOnAllTransports(this, 'channelattach2', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattach2', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -262,7 +262,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL4 * @spec RTL5 */ - Helper.testOnAllTransports( + Helper.testOnAllTransportsAndProtocols( this, 'channelattach3', function (realtimeOpts) { @@ -303,7 +303,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @spec RTL4d */ - Helper.testOnAllTransports(this, 'channelattachempty', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattachempty', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -338,7 +338,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @spec RTL4d */ - Helper.testOnAllTransports(this, 'channelattachinvalid', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattachinvalid', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -379,7 +379,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @spec RTL6 */ - Helper.testOnAllTransports(this, 'publish_no_attach', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'publish_no_attach', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -409,7 +409,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @specpartial RTL6b - callback which is called with an error */ - Helper.testOnAllTransports(this, 'channelattach_publish_invalid', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattach_publish_invalid', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -443,7 +443,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * * @nospec */ - Helper.testOnAllTransports(this, 'channelattach_invalid_twice', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'channelattach_invalid_twice', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -538,7 +538,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL4k1 * @spec RTL4m */ - Helper.testOnAllTransports(this, 'attachWithChannelParamsBasicChannelsGet', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'attachWithChannelParamsBasicChannelsGet', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'attachWithChannelParamsBasicChannelsGet'; @@ -601,7 +601,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL4m * @spec RTL16 */ - Helper.testOnAllTransports(this, 'attachWithChannelParamsBasicSetOptions', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'attachWithChannelParamsBasicSetOptions', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'attachWithChannelParamsBasicSetOptions'; @@ -656,7 +656,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL16 * @spec RTL7c */ - Helper.testOnAllTransports(this, 'subscribeAfterSetOptions', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'subscribeAfterSetOptions', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'subscribeAfterSetOptions'; @@ -732,7 +732,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); /** @spec RTL16a */ - Helper.testOnAllTransports(this, 'setOptionsCallbackBehaviour', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'setOptionsCallbackBehaviour', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'setOptionsCallbackBehaviour'; @@ -814,61 +814,65 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * Verify modes is ignored when params.modes is present * @nospec */ - Helper.testOnAllTransports(this, 'attachWithChannelParamsModesAndChannelModes', function (realtimeOpts) { - return function (done) { - const helper = this.test.helper; - var testName = 'attachWithChannelParamsModesAndChannelModes'; - try { - var realtime = helper.AblyRealtime(realtimeOpts); - realtime.connection.on('connected', function () { - var paramsModes = ['presence', 'subscribe']; - var params = { - modes: paramsModes.join(','), - }; - var channelOptions = { - params: params, - modes: ['publish', 'presence_subscribe'], - }; - var channel = realtime.channels.get(testName, channelOptions); - Helper.whenPromiseSettles(channel.attach(), function (err) { - if (err) { - helper.closeAndFinish(done, realtime, err); - return; - } - try { - helper.recordPrivateApi('read.channel.channelOptions'); - expect(channel.channelOptions).to.deep.equal(channelOptions, 'Check requested channel options'); - expect(channel.params).to.deep.equal(params, 'Check result params'); - expect(channel.modes).to.deep.equal(paramsModes, 'Check result modes'); - } catch (err) { - helper.closeAndFinish(done, realtime, err); - return; - } + Helper.testOnAllTransportsAndProtocols( + this, + 'attachWithChannelParamsModesAndChannelModes', + function (realtimeOpts) { + return function (done) { + const helper = this.test.helper; + var testName = 'attachWithChannelParamsModesAndChannelModes'; + try { + var realtime = helper.AblyRealtime(realtimeOpts); + realtime.connection.on('connected', function () { + var paramsModes = ['presence', 'subscribe']; + var params = { + modes: paramsModes.join(','), + }; + var channelOptions = { + params: params, + modes: ['publish', 'presence_subscribe'], + }; + var channel = realtime.channels.get(testName, channelOptions); + Helper.whenPromiseSettles(channel.attach(), function (err) { + if (err) { + helper.closeAndFinish(done, realtime, err); + return; + } + try { + helper.recordPrivateApi('read.channel.channelOptions'); + expect(channel.channelOptions).to.deep.equal(channelOptions, 'Check requested channel options'); + expect(channel.params).to.deep.equal(params, 'Check result params'); + expect(channel.modes).to.deep.equal(paramsModes, 'Check result modes'); + } catch (err) { + helper.closeAndFinish(done, realtime, err); + return; + } - var testRealtime = helper.AblyRealtime(); - testRealtime.connection.on('connected', function () { - var testChannel = testRealtime.channels.get(testName); - async.series( - [ - checkCanSubscribe(channel, testChannel), - checkCanEnterPresence(channel), - checkCantPublish(channel), - checkCantPresenceSubscribe(channel, testChannel), - ], - function (err) { - testRealtime.close(); - helper.closeAndFinish(done, realtime, err); - }, - ); + var testRealtime = helper.AblyRealtime(); + testRealtime.connection.on('connected', function () { + var testChannel = testRealtime.channels.get(testName); + async.series( + [ + checkCanSubscribe(channel, testChannel), + checkCanEnterPresence(channel), + checkCantPublish(channel), + checkCantPresenceSubscribe(channel, testChannel), + ], + function (err) { + testRealtime.close(); + helper.closeAndFinish(done, realtime, err); + }, + ); + }); }); }); - }); - helper.monitorConnection(done, realtime); - } catch (err) { - helper.closeAndFinish(done, realtime, err); - } - }; - }); + helper.monitorConnection(done, realtime); + } catch (err) { + helper.closeAndFinish(done, realtime, err); + } + }; + }, + ); /** * No spec items found for 'modes' property behavior (like preventing publish, entering presence, presence subscription) @@ -877,7 +881,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL4l * @spec RTL4m */ - Helper.testOnAllTransports(this, 'attachWithChannelModes', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'attachWithChannelModes', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'attachWithChannelModes'; @@ -937,7 +941,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL4l * @spec RTL4m */ - Helper.testOnAllTransports(this, 'attachWithChannelParamsDeltaAndModes', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'attachWithChannelParamsDeltaAndModes', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var testName = 'attachWithChannelParamsDeltaAndModes'; @@ -1259,7 +1263,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); var transport = realtime.connection.connectionManager.activeProtocol.getTransport(); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); transport.onProtocolMessage( createPM({ @@ -1309,7 +1313,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async } helper.recordPrivateApi('call.Platform.nextTick'); Ably.Realtime.Platform.Config.nextTick(function () { - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); transport.onProtocolMessage( createPM({ @@ -1360,7 +1364,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); var transport = realtime.connection.connectionManager.activeProtocol.getTransport(); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); transport.onProtocolMessage( createPM({ @@ -1408,7 +1412,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); var transport = realtime.connection.connectionManager.activeProtocol.getTransport(); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); transport.onProtocolMessage( createPM({ @@ -1614,7 +1618,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async setTimeout(function () { helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); var transport = realtime.connection.connectionManager.activeProtocol.getTransport(); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); transport.onProtocolMessage(createPM({ action: 11, channel: channelName })); }, 0); @@ -1861,5 +1865,131 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async await helper.closeAndFinishAsync(realtime); }); + + /** @spec RTL4c1 */ + it('set channelSerial field for ATTACH ProtocolMessage if available', async function () { + const helper = this.test.helper; + const realtime = helper.AblyRealtime(); + + await helper.monitorConnectionAsync(async () => { + const channel = realtime.channels.get('set_channelSerial_on_attach'); + await realtime.connection.once('connected'); + await channel.attach(); + + // Publish a message to get the channelSerial from it + const messageReceivedPromise = new Promise((resolve, reject) => { + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + const transport = realtime.connection.connectionManager.activeProtocol.getTransport(); + const onProtocolMessageOriginal = transport.onProtocolMessage; + + helper.recordPrivateApi('replace.transport.onProtocolMessage'); + transport.onProtocolMessage = function (msg) { + if (msg.action === 15) { + // MESSAGE + resolve(msg); + } + + helper.recordPrivateApi('call.transport.onProtocolMessage'); + onProtocolMessageOriginal.call(this, msg); + }; + }); + await channel.publish('event', 'test'); + + const receivedMessage = await messageReceivedPromise; + helper.recordPrivateApi('read.ProtocolMessage.channelSerial'); + const receivedChannelSerial = receivedMessage.channelSerial; + + // After the disconnect, on reconnect, spy on transport.send to check sent channelSerial + const promiseCheck = new Promise((resolve, reject) => { + helper.recordPrivateApi('listen.connectionManager.transport.pending'); + realtime.connection.connectionManager.once('transport.pending', function (transport) { + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + const sendOriginal = transport.send; + + helper.recordPrivateApi('replace.transport.send'); + transport.send = function (msg) { + if (msg.action === 10) { + // ATTACH + try { + helper.recordPrivateApi('read.ProtocolMessage.channelSerial'); + expect(msg.channelSerial).to.equal(receivedChannelSerial); + resolve(); + } catch (error) { + reject(error); + } + } + + helper.recordPrivateApi('call.transport.send'); + sendOriginal.call(this, msg); + }; + }); + }); + + // Disconnect the transport (will automatically reconnect and resume) + helper.recordPrivateApi('call.connectionManager.disconnectAllTransports'); + realtime.connection.connectionManager.disconnectAllTransports(); + + await promiseCheck; + }, realtime); + + await helper.closeAndFinishAsync(realtime); + }); + + /** @spec RTL15b */ + it('channel.properties.channelSerial is updated with channelSerial from latest message', async function () { + const helper = this.test.helper; + const realtime = helper.AblyRealtime({ clientId: 'me' }); + + await helper.monitorConnectionAsync(async () => { + const channel = realtime.channels.get('update_channelSerial_on_message'); + await realtime.connection.once('connected'); + + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + const messagesToUpdateChannelSerial = [ + createPM({ + action: 11, // ATTACHED + channel: channel.name, + channelSerial: 'ATTACHED', + }), + createPM({ + action: 15, // MESSAGE + channel: channel.name, + channelSerial: 'MESSAGE', + messages: [{ name: 'foo', data: 'bar' }], + }), + createPM({ + action: 14, // PRESENCE + channel: channel.name, + channelSerial: 'PRESENCE', + }), + createPM({ + action: 19, // OBJECT + channel: channel.name, + channelSerial: 'OBJECT', + }), + createPM({ + action: 21, // ANNOTATION + channel: channel.name, + channelSerial: 'ANNOTATION', + }), + ]; + + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + const transport = realtime.connection.connectionManager.activeProtocol.getTransport(); + + for (const msg of messagesToUpdateChannelSerial) { + helper.recordPrivateApi('call.transport.onProtocolMessage'); + transport.onProtocolMessage(msg); + + // wait until next event loop so any async ops get resolved and channel serial gets updated on a channel + await new Promise((res) => setTimeout(res, 0)); + helper.recordPrivateApi('read.channel.properties.channelSerial'); + helper.recordPrivateApi('read.ProtocolMessage.channelSerial'); + expect(channel.properties.channelSerial).to.equal(msg.channelSerial); + } + }, realtime); + + await helper.closeAndFinishAsync(realtime); + }); }); }); diff --git a/test/realtime/connection.test.js b/test/realtime/connection.test.js index c8551286c6..82c506ae50 100644 --- a/test/realtime/connection.test.js +++ b/test/realtime/connection.test.js @@ -2,7 +2,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async, chai) { var expect = chai.expect; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); describe('realtime/connection', function () { this.timeout(60 * 1000); @@ -336,7 +336,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); helper.recordPrivateApi('call.transport.onProtocolMessage'); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); connectionManager.activeProtocol.getTransport().onProtocolMessage( createPM({ action: 4, diff --git a/test/realtime/crypto.test.js b/test/realtime/crypto.test.js index fedeac5802..1ec6add9ef 100644 --- a/test/realtime/crypto.test.js +++ b/test/realtime/crypto.test.js @@ -461,7 +461,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async } function single_send(done, helper, realtimeOpts, keyLength) { - // the _128 and _256 variants both call this so it makes more sense for this to be the parameterisedTestTitle instead of that set by testOnAllTransports + // the _128 and _256 variants both call this so it makes more sense for this to be the parameterisedTestTitle instead of that set by testOnAllTransportsAndProtocols helper = helper.withParameterisedTestTitle('single_send'); if (!Crypto) { @@ -509,14 +509,14 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async // Publish and subscribe, various transport, 128 and 256-bit /** @specpartial RSL5b - test aes 128 */ - Helper.testOnAllTransports(this, 'single_send_128', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'single_send_128', function (realtimeOpts) { return function (done) { single_send(done, this.test.helper, realtimeOpts, 128); }; }); /** @specpartial RSL5b - test aes 256 */ - Helper.testOnAllTransports(this, 'single_send_256', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'single_send_256', function (realtimeOpts) { return function (done) { single_send(done, this.test.helper, realtimeOpts, 256); }; diff --git a/test/realtime/failure.test.js b/test/realtime/failure.test.js index 4f214f32f7..f397bd7fc8 100644 --- a/test/realtime/failure.test.js +++ b/test/realtime/failure.test.js @@ -3,7 +3,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async, chai) { var expect = chai.expect; var noop = function () {}; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); describe('realtime/failure', function () { this.timeout(60 * 1000); @@ -619,7 +619,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async }); /** @specpartial RTN14d - last sentence: check that if we received a 5xx disconnected, when we try again we use a fallback host */ - Helper.testOnAllTransports(this, 'try_fallback_hosts_on_placement_constraint', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'try_fallback_hosts_on_placement_constraint', function (realtimeOpts) { return function (done) { const helper = this.test.helper; /* Use the echoserver as a fallback host because it doesn't support @@ -644,7 +644,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async helper.closeAndFinish(done, realtime); }); }); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); helper.recordPrivateApi('call.transport.onProtocolMessage'); connectionManager.activeProtocol.getTransport().onProtocolMessage( diff --git a/test/realtime/message.test.js b/test/realtime/message.test.js index fd34c58788..8d5a81b35d 100644 --- a/test/realtime/message.test.js +++ b/test/realtime/message.test.js @@ -3,7 +3,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async, chai) { var expect = chai.expect; let config = Ably.Realtime.Platform.Config; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); var Message = Ably.Realtime.Message; var publishIntervalHelper = function (currentMessageNum, channel, dataFn, onPublish) { @@ -79,7 +79,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * Test publishes in quick succession (on successive ticks of the event loop) * @spec RTL6b */ - Helper.testOnAllTransports(this, 'publishfast', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'publishfast', function (realtimeOpts) { return function (done) { const helper = this.test.helper; try { @@ -146,7 +146,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL6c2 * @specpartial RTL3d - test processing queued messages */ - Helper.testOnAllTransports(this, 'publishQueued', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'publishQueued', function (realtimeOpts) { return function (done) { var helper = this.test.helper, txRealtime, @@ -629,7 +629,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec RTL6 * @spec RTL6b */ - Helper.testOnAllTransports(this, 'publish', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'publish', function (realtimeOpts) { return function (done) { const helper = this.test.helper; var count = 10; @@ -1147,7 +1147,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async helper.recordPrivateApi('write.connectionManager.connectionDetails.maxMessageSize'); connectionDetails.maxMessageSize = 64; helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.transport.onProtocolMessage'); connectionManager.activeProtocol.getTransport().onProtocolMessage( createPM({ diff --git a/test/realtime/objects.test.js b/test/realtime/objects.test.js new file mode 100644 index 0000000000..220694199c --- /dev/null +++ b/test/realtime/objects.test.js @@ -0,0 +1,4975 @@ +'use strict'; + +define(['ably', 'shared_helper', 'chai', 'objects', 'objects_helper'], function ( + Ably, + Helper, + chai, + ObjectsPlugin, + ObjectsHelper, +) { + const expect = chai.expect; + const BufferUtils = Ably.Realtime.Platform.BufferUtils; + const Utils = Ably.Realtime.Utils; + const MessageEncoding = Ably.Realtime._MessageEncoding; + const createPM = Ably.makeProtocolMessageFromDeserialized({ ObjectsPlugin }); + const objectsFixturesChannel = 'objects_fixtures'; + const nextTick = Ably.Realtime.Platform.Config.nextTick; + const gcIntervalOriginal = ObjectsPlugin.Objects._DEFAULTS.gcInterval; + const gcGracePeriodOriginal = ObjectsPlugin.Objects._DEFAULTS.gcGracePeriod; + + function RealtimeWithObjects(helper, options) { + return helper.AblyRealtime({ ...options, plugins: { Objects: ObjectsPlugin } }); + } + + function channelOptionsWithObjects(options) { + return { + ...options, + modes: ['OBJECT_SUBSCRIBE', 'OBJECT_PUBLISH'], + }; + } + + function expectInstanceOf(object, className, msg) { + // esbuild changes the name for classes with static method to include an underscore as prefix. + // so LiveMap becomes _LiveMap. we account for it here. + expect(object.constructor.name).to.match(new RegExp(`_?${className}`), msg); + } + + function forScenarios(thisInDescribe, scenarios, testFn) { + for (const scenario of scenarios) { + if (scenario.allTransportsAndProtocols) { + Helper.testOnAllTransportsAndProtocols( + thisInDescribe, + scenario.description, + function (options, channelName) { + return async function () { + const helper = this.test.helper; + await testFn(helper, scenario, options, channelName); + }; + }, + scenario.skip, + scenario.only, + ); + } else { + const itFn = scenario.skip ? it.skip : scenario.only ? it.only : it; + + itFn(scenario.description, async function () { + const helper = this.test.helper; + await testFn(helper, scenario, {}, scenario.description); + }); + } + } + } + + function lexicoTimeserial(seriesId, timestamp, counter, index) { + const paddedTimestamp = timestamp.toString().padStart(14, '0'); + const paddedCounter = counter.toString().padStart(3, '0'); + const paddedIndex = index != null ? index.toString().padStart(3, '0') : undefined; + + // Example: + // + // 01726585978590-001@abcdefghij:001 + // |____________| |_| |________| |_| + // | | | | + // timestamp counter seriesId idx + return `${paddedTimestamp}-${paddedCounter}@${seriesId}` + (paddedIndex ? `:${paddedIndex}` : ''); + } + + async function expectToThrowAsync(fn, errorStr) { + let savedError; + try { + await fn(); + } catch (error) { + expect(error.message).to.have.string(errorStr); + savedError = error; + } + expect(savedError, 'Expected async function to throw an error').to.exist; + + return savedError; + } + + function objectMessageFromValues(values) { + return ObjectsPlugin.ObjectMessage.fromValues(values, Utils, MessageEncoding); + } + + async function waitForMapKeyUpdate(map, key) { + return new Promise((resolve) => { + const { unsubscribe } = map.subscribe(({ update }) => { + if (update[key]) { + unsubscribe(); + resolve(); + } + }); + }); + } + + async function waitForCounterUpdate(counter) { + return new Promise((resolve) => { + const { unsubscribe } = counter.subscribe(() => { + unsubscribe(); + resolve(); + }); + }); + } + + async function waitForObjectOperation(helper, client, waitForAction) { + return new Promise((resolve, reject) => { + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + const transport = client.connection.connectionManager.activeProtocol.getTransport(); + const onProtocolMessageOriginal = transport.onProtocolMessage; + + helper.recordPrivateApi('replace.transport.onProtocolMessage'); + transport.onProtocolMessage = function (message) { + try { + helper.recordPrivateApi('call.transport.onProtocolMessage'); + onProtocolMessageOriginal.call(transport, message); + + if (message.action === 19 && message.state[0]?.operation?.action === waitForAction) { + helper.recordPrivateApi('replace.transport.onProtocolMessage'); + transport.onProtocolMessage = onProtocolMessageOriginal; + resolve(); + } + } catch (err) { + reject(err); + } + }; + }); + } + + async function waitForObjectSync(helper, client) { + return new Promise((resolve, reject) => { + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + const transport = client.connection.connectionManager.activeProtocol.getTransport(); + const onProtocolMessageOriginal = transport.onProtocolMessage; + + helper.recordPrivateApi('replace.transport.onProtocolMessage'); + transport.onProtocolMessage = function (message) { + try { + helper.recordPrivateApi('call.transport.onProtocolMessage'); + onProtocolMessageOriginal.call(transport, message); + + if (message.action === 20) { + helper.recordPrivateApi('replace.transport.onProtocolMessage'); + transport.onProtocolMessage = onProtocolMessageOriginal; + resolve(); + } + } catch (err) { + reject(err); + } + }; + }); + } + + /** + * The channel with fixture data may not yet be populated by REST API requests made by ObjectsHelper. + * This function waits for a channel to have all keys set. + */ + async function waitFixtureChannelIsReady(client) { + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; + const expectedKeys = ObjectsHelper.fixtureRootKeys(); + + await channel.attach(); + const root = await objects.getRoot(); + + await Promise.all(expectedKeys.map((key) => (root.get(key) ? undefined : waitForMapKeyUpdate(root, key)))); + } + + describe('realtime/objects', function () { + this.timeout(60 * 1000); + + before(function (done) { + const helper = Helper.forHook(this); + + helper.setupApp(function (err) { + if (err) { + done(err); + return; + } + + new ObjectsHelper(helper) + .initForChannel(objectsFixturesChannel) + .then(done) + .catch((err) => done(err)); + }); + }); + + describe('Realtime without Objects plugin', () => { + /** @nospec */ + it("throws an error when attempting to access the channel's `objects` property", async function () { + const helper = this.test.helper; + const client = helper.AblyRealtime({ autoConnect: false }); + const channel = client.channels.get('channel'); + expect(() => channel.objects).to.throw('Objects plugin not provided'); + }); + + /** @nospec */ + it(`doesn't break when it receives an OBJECT ProtocolMessage`, async function () { + const helper = this.test.helper; + const objectsHelper = new ObjectsHelper(helper); + const testClient = helper.AblyRealtime(); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const testChannel = testClient.channels.get('channel'); + await testChannel.attach(); + + const receivedMessagePromise = new Promise((resolve) => testChannel.subscribe(resolve)); + + const publishClient = helper.AblyRealtime(); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + // inject OBJECT message that should be ignored and not break anything without the plugin + await objectsHelper.processObjectOperationMessageOnChannel({ + channel: testChannel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'stringKey', data: { string: 'stringValue' } })], + }); + + const publishChannel = publishClient.channels.get('channel'); + await publishChannel.publish(null, 'test'); + + // regular message subscriptions should still work after processing OBJECT_SYNC message without the plugin + await receivedMessagePromise; + }, publishClient); + }, testClient); + }); + + /** @nospec */ + it(`doesn't break when it receives an OBJECT_SYNC ProtocolMessage`, async function () { + const helper = this.test.helper; + const objectsHelper = new ObjectsHelper(helper); + const testClient = helper.AblyRealtime(); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const testChannel = testClient.channels.get('channel'); + await testChannel.attach(); + + const receivedMessagePromise = new Promise((resolve) => testChannel.subscribe(resolve)); + + const publishClient = helper.AblyRealtime(); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + // inject OBJECT_SYNC message that should be ignored and not break anything without the plugin + await objectsHelper.processObjectStateMessageOnChannel({ + channel: testChannel, + syncSerial: 'serial:', + state: [ + objectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + }), + ], + }); + + const publishChannel = publishClient.channels.get('channel'); + await publishChannel.publish(null, 'test'); + + // regular message subscriptions should still work after processing OBJECT_SYNC message without the plugin + await receivedMessagePromise; + }, publishClient); + }, testClient); + }); + }); + + describe('Realtime with Objects plugin', () => { + /** @nospec */ + it("returns Objects class instance when accessing channel's `objects` property", async function () { + const helper = this.test.helper; + const client = RealtimeWithObjects(helper, { autoConnect: false }); + const channel = client.channels.get('channel'); + expectInstanceOf(channel.objects, 'Objects'); + }); + + /** @nospec */ + it('getRoot() returns LiveMap instance', async function () { + const helper = this.test.helper; + const client = RealtimeWithObjects(helper); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + expectInstanceOf(root, 'LiveMap', 'root object should be of LiveMap type'); + }, client); + }); + + /** @nospec */ + it('getRoot() returns LiveObject with id "root"', async function () { + const helper = this.test.helper; + const client = RealtimeWithObjects(helper); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + helper.recordPrivateApi('call.LiveObject.getObjectId'); + expect(root.getObjectId()).to.equal('root', 'root object should have an object id "root"'); + }, client); + }); + + /** @nospec */ + it('getRoot() returns empty root when no objects exist on a channel', async function () { + const helper = this.test.helper; + const client = RealtimeWithObjects(helper); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + expect(root.size()).to.equal(0, 'Check root has no keys'); + }, client); + }); + + /** @nospec */ + it('getRoot() waits for initial OBJECT_SYNC to be completed before resolving', async function () { + const helper = this.test.helper; + const client = RealtimeWithObjects(helper); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; + + const getRootPromise = objects.getRoot(); + + let getRootResolved = false; + getRootPromise.then(() => { + getRootResolved = true; + }); + + // give a chance for getRoot() to resolve and proc its handler. it should not + helper.recordPrivateApi('call.Platform.nextTick'); + await new Promise((res) => nextTick(res)); + expect(getRootResolved, 'Check getRoot() is not resolved until OBJECT_SYNC sequence is completed').to.be + .false; + + await channel.attach(); + + // should resolve eventually after attach + await getRootPromise; + }, client); + }); + + /** @nospec */ + it('getRoot() resolves immediately when OBJECT_SYNC sequence is completed', async function () { + const helper = this.test.helper; + const client = RealtimeWithObjects(helper); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + // wait for sync sequence to complete by accessing root for the first time + await objects.getRoot(); + + let resolvedImmediately = false; + objects.getRoot().then(() => { + resolvedImmediately = true; + }); + + // wait for next tick for getRoot() handler to process + helper.recordPrivateApi('call.Platform.nextTick'); + await new Promise((res) => nextTick(res)); + + expect(resolvedImmediately, 'Check getRoot() is resolved on next tick').to.be.true; + }, client); + }); + + /** @nospec */ + it('getRoot() waits for OBJECT_SYNC with empty cursor before resolving', async function () { + const helper = this.test.helper; + const objectsHelper = new ObjectsHelper(helper); + const client = RealtimeWithObjects(helper); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + // wait for initial sync sequence to complete + await objects.getRoot(); + + // inject OBJECT_SYNC message to emulate start of a new sequence + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + // have cursor so client awaits for additional OBJECT_SYNC messages + syncSerial: 'serial:cursor', + }); + + let getRootResolved = false; + let root; + objects.getRoot().then((value) => { + getRootResolved = true; + root = value; + }); + + // wait for next tick to check that getRoot() promise handler didn't proc + helper.recordPrivateApi('call.Platform.nextTick'); + await new Promise((res) => nextTick(res)); + + expect(getRootResolved, 'Check getRoot() is not resolved while OBJECT_SYNC is in progress').to.be.false; + + // inject final OBJECT_SYNC message + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + // no cursor to indicate the end of OBJECT_SYNC messages + syncSerial: 'serial:', + state: [ + objectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + initialEntries: { key: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { number: 1 } } }, + }), + ], + }); + + // wait for next tick for getRoot() handler to process + helper.recordPrivateApi('call.Platform.nextTick'); + await new Promise((res) => nextTick(res)); + + expect(getRootResolved, 'Check getRoot() is resolved when OBJECT_SYNC sequence has ended').to.be.true; + expect(root.get('key')).to.equal(1, 'Check new root after OBJECT_SYNC sequence has expected key'); + }, client); + }); + + const primitiveKeyData = [ + { key: 'stringKey', data: { string: 'stringValue' } }, + { key: 'emptyStringKey', data: { string: '' } }, + { key: 'bytesKey', data: { bytes: 'eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9' } }, + { key: 'emptyBytesKey', data: { bytes: '' } }, + { key: 'maxSafeIntegerKey', data: { number: Number.MAX_SAFE_INTEGER } }, + { key: 'negativeMaxSafeIntegerKey', data: { number: -Number.MAX_SAFE_INTEGER } }, + { key: 'numberKey', data: { number: 1 } }, + { key: 'zeroKey', data: { number: 0 } }, + { key: 'trueKey', data: { boolean: true } }, + { key: 'falseKey', data: { boolean: false } }, + ]; + const primitiveMapsFixtures = [ + { name: 'emptyMap' }, + { + name: 'valuesMap', + entries: primitiveKeyData.reduce((acc, v) => { + acc[v.key] = { data: v.data }; + return acc; + }, {}), + restData: primitiveKeyData.reduce((acc, v) => { + acc[v.key] = v.data; + return acc; + }, {}), + }, + ]; + const countersFixtures = [ + { name: 'emptyCounter' }, + { name: 'zeroCounter', count: 0 }, + { name: 'valueCounter', count: 10 }, + { name: 'negativeValueCounter', count: -10 }, + { name: 'maxSafeIntegerCounter', count: Number.MAX_SAFE_INTEGER }, + { name: 'negativeMaxSafeIntegerCounter', count: -Number.MAX_SAFE_INTEGER }, + ]; + + const objectSyncSequenceScenarios = [ + { + allTransportsAndProtocols: true, + description: 'OBJECT_SYNC sequence builds object tree on channel attachment', + action: async (ctx) => { + const { client } = ctx; + + await waitFixtureChannelIsReady(client); + + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + const counterKeys = ['emptyCounter', 'initialValueCounter', 'referencedCounter']; + const mapKeys = ['emptyMap', 'referencedMap', 'valuesMap']; + const rootKeysCount = counterKeys.length + mapKeys.length; + + expect(root, 'Check getRoot() is resolved when OBJECT_SYNC sequence ends').to.exist; + expect(root.size()).to.equal(rootKeysCount, 'Check root has correct number of keys'); + + counterKeys.forEach((key) => { + const counter = root.get(key); + expect(counter, `Check counter at key="${key}" in root exists`).to.exist; + expectInstanceOf(counter, 'LiveCounter', `Check counter at key="${key}" in root is of type LiveCounter`); + }); + + mapKeys.forEach((key) => { + const map = root.get(key); + expect(map, `Check map at key="${key}" in root exists`).to.exist; + expectInstanceOf(map, 'LiveMap', `Check map at key="${key}" in root is of type LiveMap`); + }); + + const valuesMap = root.get('valuesMap'); + const valueMapKeys = [ + 'stringKey', + 'emptyStringKey', + 'bytesKey', + 'emptyBytesKey', + 'numberKey', + 'zeroKey', + 'trueKey', + 'falseKey', + 'mapKey', + ]; + expect(valuesMap.size()).to.equal(valueMapKeys.length, 'Check nested map has correct number of keys'); + valueMapKeys.forEach((key) => { + const value = valuesMap.get(key); + expect(value, `Check value at key="${key}" in nested map exists`).to.exist; + }); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'OBJECT_SYNC sequence builds object tree with all operations applied', + action: async (ctx) => { + const { root, objects, helper, clientOptions, channelName } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + + // MAP_CREATE + const map = await objects.createMap({ shouldStay: 'foo', shouldDelete: 'bar' }); + // COUNTER_CREATE + const counter = await objects.createCounter(1); + + await Promise.all([root.set('map', map), root.set('counter', counter), objectsCreatedPromise]); + + const operationsAppliedPromise = Promise.all([ + waitForMapKeyUpdate(map, 'anotherKey'), + waitForMapKeyUpdate(map, 'shouldDelete'), + waitForCounterUpdate(counter), + ]); + + await Promise.all([ + // MAP_SET + map.set('anotherKey', 'baz'), + // MAP_REMOVE + map.remove('shouldDelete'), + // COUNTER_INC + counter.increment(10), + operationsAppliedPromise, + ]); + + // create a new client and check it syncs with the aggregated data + const client2 = RealtimeWithObjects(helper, clientOptions); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const channel2 = client2.channels.get(channelName, channelOptionsWithObjects()); + const objects2 = channel2.objects; + + await channel2.attach(); + const root2 = await objects2.getRoot(); + + expect(root2.get('counter'), 'Check counter exists').to.exist; + expect(root2.get('counter').value()).to.equal(11, 'Check counter has correct value'); + + expect(root2.get('map'), 'Check map exists').to.exist; + expect(root2.get('map').size()).to.equal(2, 'Check map has correct number of keys'); + expect(root2.get('map').get('shouldStay')).to.equal( + 'foo', + 'Check map has correct value for "shouldStay" key', + ); + expect(root2.get('map').get('anotherKey')).to.equal( + 'baz', + 'Check map has correct value for "anotherKey" key', + ); + expect(root2.get('map').get('shouldDelete'), 'Check map does not have "shouldDelete" key').to.not.exist; + }, client2); + }, + }, + + { + description: 'OBJECT_SYNC sequence does not change references to existing objects', + action: async (ctx) => { + const { root, objects, helper, channel } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + + const map = await objects.createMap(); + const counter = await objects.createCounter(); + await Promise.all([root.set('map', map), root.set('counter', counter), objectsCreatedPromise]); + await channel.detach(); + + // wait for the actual OBJECT_SYNC message to confirm it was received and processed + const objectSyncPromise = waitForObjectSync(helper, channel.client); + await channel.attach(); + await objectSyncPromise; + + const newRootRef = await channel.objects.getRoot(); + const newMapRef = newRootRef.get('map'); + const newCounterRef = newRootRef.get('counter'); + + expect(newRootRef).to.equal(root, 'Check root reference is the same after OBJECT_SYNC sequence'); + expect(newMapRef).to.equal(map, 'Check map reference is the same after OBJECT_SYNC sequence'); + expect(newCounterRef).to.equal(counter, 'Check counter reference is the same after OBJECT_SYNC sequence'); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveCounter is initialized with initial value from OBJECT_SYNC sequence', + action: async (ctx) => { + const { client } = ctx; + + await waitFixtureChannelIsReady(client); + + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + const counters = [ + { key: 'emptyCounter', value: 0 }, + { key: 'initialValueCounter', value: 10 }, + { key: 'referencedCounter', value: 20 }, + ]; + + counters.forEach((x) => { + const counter = root.get(x.key); + expect(counter.value()).to.equal(x.value, `Check counter at key="${x.key}" in root has correct value`); + }); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveMap is initialized with initial value from OBJECT_SYNC sequence', + action: async (ctx) => { + const { helper, client } = ctx; + + await waitFixtureChannelIsReady(client); + + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + const emptyMap = root.get('emptyMap'); + expect(emptyMap.size()).to.equal(0, 'Check empty map in root has no keys'); + + const referencedMap = root.get('referencedMap'); + expect(referencedMap.size()).to.equal(1, 'Check referenced map in root has correct number of keys'); + + const counterFromReferencedMap = referencedMap.get('counterKey'); + expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); + + const valuesMap = root.get('valuesMap'); + expect(valuesMap.size()).to.equal(9, 'Check values map in root has correct number of keys'); + + expect(valuesMap.get('stringKey')).to.equal('stringValue', 'Check values map has correct string value key'); + expect(valuesMap.get('emptyStringKey')).to.equal('', 'Check values map has correct empty string value key'); + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual( + valuesMap.get('bytesKey'), + BufferUtils.base64Decode('eyJwcm9kdWN0SWQiOiAiMDAxIiwgInByb2R1Y3ROYW1lIjogImNhciJ9'), + ), + 'Check values map has correct bytes value key', + ).to.be.true; + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual(valuesMap.get('emptyBytesKey'), BufferUtils.base64Decode('')), + 'Check values map has correct empty bytes value key', + ).to.be.true; + expect(valuesMap.get('numberKey')).to.equal(1, 'Check values map has correct number value key'); + expect(valuesMap.get('zeroKey')).to.equal(0, 'Check values map has correct zero number value key'); + expect(valuesMap.get('trueKey')).to.equal(true, `Check values map has correct 'true' value key`); + expect(valuesMap.get('falseKey')).to.equal(false, `Check values map has correct 'false' value key`); + + const mapFromValuesMap = valuesMap.get('mapKey'); + expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveMap can reference the same object in their keys', + action: async (ctx) => { + const { client } = ctx; + + await waitFixtureChannelIsReady(client); + + const channel = client.channels.get(objectsFixturesChannel, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + const referencedCounter = root.get('referencedCounter'); + const referencedMap = root.get('referencedMap'); + const valuesMap = root.get('valuesMap'); + + const counterFromReferencedMap = referencedMap.get('counterKey'); + expect(counterFromReferencedMap, 'Check nested counter exists at a key in a map').to.exist; + expectInstanceOf(counterFromReferencedMap, 'LiveCounter', 'Check nested counter is of type LiveCounter'); + expect(counterFromReferencedMap).to.equal( + referencedCounter, + 'Check nested counter is the same object instance as counter on the root', + ); + expect(counterFromReferencedMap.value()).to.equal(20, 'Check nested counter has correct value'); + + const mapFromValuesMap = valuesMap.get('mapKey'); + expect(mapFromValuesMap, 'Check nested map exists at a key in a map').to.exist; + expectInstanceOf(mapFromValuesMap, 'LiveMap', 'Check nested map is of type LiveMap'); + expect(mapFromValuesMap.size()).to.equal(1, 'Check nested map has correct number of keys'); + expect(mapFromValuesMap).to.equal( + referencedMap, + 'Check nested map is the same object instance as map on the root', + ); + }, + }, + + { + description: 'OBJECT_SYNC sequence with object state "tombstone" property creates tombstoned object', + action: async (ctx) => { + const { root, objectsHelper, channel } = ctx; + + const mapId = objectsHelper.fakeMapObjectId(); + const counterId = objectsHelper.fakeCounterObjectId(); + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:', // empty serial so sync sequence ends immediately + // add object states with tombstone=true + state: [ + objectsHelper.mapObject({ + objectId: mapId, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialEntries: {}, + }), + objectsHelper.counterObject({ + objectId: counterId, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialCount: 1, + }), + objectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + initialEntries: { + map: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: mapId } }, + counter: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId } }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'bar' } }, + }, + }), + ], + }); + + expect( + root.get('map'), + 'Check map does not exist on root after OBJECT_SYNC with "tombstone=true" for a map object', + ).to.not.exist; + expect( + root.get('counter'), + 'Check counter does not exist on root after OBJECT_SYNC with "tombstone=true" for a counter object', + ).to.not.exist; + // control check that OBJECT_SYNC was applied at all + expect(root.get('foo'), 'Check property exists on root after OBJECT_SYNC').to.exist; + }, + }, + + { + allTransportsAndProtocols: true, + description: 'OBJECT_SYNC sequence with object state "tombstone" property deletes existing object', + action: async (ctx) => { + const { root, objectsHelper, channelName, channel } = ctx; + + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); + const { objectId: counterId } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: objectsHelper.counterCreateRestOp({ number: 1 }), + }); + await counterCreatedPromise; + + expect( + root.get('counter'), + 'Check counter exists on root before OBJECT_SYNC sequence with "tombstone=true"', + ).to.exist; + + // inject an OBJECT_SYNC message where a counter is now tombstoned + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:', // empty serial so sync sequence ends immediately + state: [ + objectsHelper.counterObject({ + objectId: counterId, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialCount: 1, + }), + objectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + initialEntries: { + counter: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId } }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'bar' } }, + }, + }), + ], + }); + + expect( + root.get('counter'), + 'Check counter does not exist on root after OBJECT_SYNC with "tombstone=true" for an existing counter object', + ).to.not.exist; + // control check that OBJECT_SYNC was applied at all + expect(root.get('foo'), 'Check property exists on root after OBJECT_SYNC').to.exist; + }, + }, + + { + allTransportsAndProtocols: true, + description: + 'OBJECT_SYNC sequence with object state "tombstone" property triggers subscription callback for existing object', + action: async (ctx) => { + const { root, objectsHelper, channelName, channel } = ctx; + + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); + const { objectId: counterId } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: objectsHelper.counterCreateRestOp({ number: 1 }), + }); + await counterCreatedPromise; + + const counterSubPromise = new Promise((resolve, reject) => + root.get('counter').subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { amount: -1 } }, + 'Check counter subscription callback is called with an expected update object after OBJECT_SYNC sequence with "tombstone=true"', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + + // inject an OBJECT_SYNC message where a counter is now tombstoned + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:', // empty serial so sync sequence ends immediately + state: [ + objectsHelper.counterObject({ + objectId: counterId, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialCount: 1, + }), + objectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + initialEntries: { + counter: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId } }, + }, + }), + ], + }); + + await counterSubPromise; + }, + }, + ]; + + const applyOperationsScenarios = [ + { + allTransportsAndProtocols: true, + description: 'can apply MAP_CREATE with primitives object operation messages', + action: async (ctx) => { + const { root, objectsHelper, channelName, helper } = ctx; + + // Objects public API allows us to check value of objects we've created based on MAP_CREATE ops + // if we assign those objects to another map (root for example), as there is no way to access those objects from the internal pool directly. + // however, in this test we put heavy focus on the data that is being created as the result of the MAP_CREATE op. + + // check no maps exist on root + primitiveMapsFixtures.forEach((fixture) => { + const key = fixture.name; + expect(root.get(key), `Check "${key}" key doesn't exist on root before applying MAP_CREATE ops`).to.not + .exist; + }); + + const mapsCreatedPromise = Promise.all(primitiveMapsFixtures.map((x) => waitForMapKeyUpdate(root, x.name))); + // create new maps and set on root + await Promise.all( + primitiveMapsFixtures.map((fixture) => + objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: fixture.name, + createOp: objectsHelper.mapCreateRestOp({ data: fixture.restData }), + }), + ), + ); + await mapsCreatedPromise; + + // check created maps + primitiveMapsFixtures.forEach((fixture) => { + const mapKey = fixture.name; + const mapObj = root.get(mapKey); + + // check all maps exist on root + expect(mapObj, `Check map at "${mapKey}" key in root exists`).to.exist; + expectInstanceOf(mapObj, 'LiveMap', `Check map at "${mapKey}" key in root is of type LiveMap`); + + // check primitive maps have correct values + expect(mapObj.size()).to.equal( + Object.keys(fixture.entries ?? {}).length, + `Check map "${mapKey}" has correct number of keys`, + ); + + Object.entries(fixture.entries ?? {}).forEach(([key, keyData]) => { + if (keyData.data.bytes != null) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual(mapObj.get(key), BufferUtils.base64Decode(keyData.data.bytes)), + `Check map "${mapKey}" has correct value for "${key}" key`, + ).to.be.true; + } else { + const valueType = typeof mapObj.get(key); + expect(mapObj.get(key)).to.equal( + keyData.data[valueType], + `Check map "${mapKey}" has correct value for "${key}" key`, + ); + } + }); + }); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'can apply MAP_CREATE with object ids object operation messages', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + const withReferencesMapKey = 'withReferencesMap'; + + // Objects public API allows us to check value of objects we've created based on MAP_CREATE ops + // if we assign those objects to another map (root for example), as there is no way to access those objects from the internal pool directly. + // however, in this test we put heavy focus on the data that is being created as the result of the MAP_CREATE op. + + // check map does not exist on root + expect( + root.get(withReferencesMapKey), + `Check "${withReferencesMapKey}" key doesn't exist on root before applying MAP_CREATE ops`, + ).to.not.exist; + + const mapCreatedPromise = waitForMapKeyUpdate(root, withReferencesMapKey); + // create map with references. need to create referenced objects first to obtain their object ids + const { objectId: referencedMapObjectId } = await objectsHelper.operationRequest( + channelName, + objectsHelper.mapCreateRestOp({ data: { stringKey: { string: 'stringValue' } } }), + ); + const { objectId: referencedCounterObjectId } = await objectsHelper.operationRequest( + channelName, + objectsHelper.counterCreateRestOp({ number: 1 }), + ); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: withReferencesMapKey, + createOp: objectsHelper.mapCreateRestOp({ + data: { + mapReference: { objectId: referencedMapObjectId }, + counterReference: { objectId: referencedCounterObjectId }, + }, + }), + }); + await mapCreatedPromise; + + // check map with references exist on root + const withReferencesMap = root.get(withReferencesMapKey); + expect(withReferencesMap, `Check map at "${withReferencesMapKey}" key in root exists`).to.exist; + expectInstanceOf( + withReferencesMap, + 'LiveMap', + `Check map at "${withReferencesMapKey}" key in root is of type LiveMap`, + ); + + // check map with references has correct values + expect(withReferencesMap.size()).to.equal( + 2, + `Check map "${withReferencesMapKey}" has correct number of keys`, + ); + + const referencedCounter = withReferencesMap.get('counterReference'); + const referencedMap = withReferencesMap.get('mapReference'); + + expect(referencedCounter, `Check counter at "counterReference" exists`).to.exist; + expectInstanceOf( + referencedCounter, + 'LiveCounter', + `Check counter at "counterReference" key is of type LiveCounter`, + ); + expect(referencedCounter.value()).to.equal(1, 'Check counter at "counterReference" key has correct value'); + + expect(referencedMap, `Check map at "mapReference" key exists`).to.exist; + expectInstanceOf(referencedMap, 'LiveMap', `Check map at "mapReference" key is of type LiveMap`); + + expect(referencedMap.size()).to.equal(1, 'Check map at "mapReference" key has correct number of keys'); + expect(referencedMap.get('stringKey')).to.equal( + 'stringValue', + 'Check map at "mapReference" key has correct "stringKey" value', + ); + }, + }, + + { + description: + 'MAP_CREATE object operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, objectsHelper, channel } = ctx; + + // need to use multiple maps as MAP_CREATE op can only be applied once to a map object + const mapIds = [ + objectsHelper.fakeMapObjectId(), + objectsHelper.fakeMapObjectId(), + objectsHelper.fakeMapObjectId(), + objectsHelper.fakeMapObjectId(), + objectsHelper.fakeMapObjectId(), + ]; + await Promise.all( + mapIds.map(async (mapId, i) => { + // send a MAP_SET op first to create a zero-value map with forged site timeserials vector (from the op), and set it on a root. + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', + state: [objectsHelper.mapSetOp({ objectId: mapId, key: 'foo', data: { string: 'bar' } })], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: mapId, data: { objectId: mapId } })], + }); + }), + ); + + // inject operations with various timeserial values + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // existing site, earlier CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, same CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, later CGO, applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied + ].entries()) { + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial, + siteCode, + state: [ + objectsHelper.mapCreateOp({ + objectId: mapIds[i], + entries: { + baz: { timeserial: serial, data: { string: 'qux' } }, + }, + }), + ], + }); + } + + // check only operations with correct timeserials were applied + const expectedMapValues = [ + { foo: 'bar' }, + { foo: 'bar' }, + { foo: 'bar', baz: 'qux' }, // applied MAP_CREATE + { foo: 'bar', baz: 'qux' }, // applied MAP_CREATE + { foo: 'bar', baz: 'qux' }, // applied MAP_CREATE + ]; + + for (const [i, mapId] of mapIds.entries()) { + const expectedMapValue = expectedMapValues[i]; + const expectedKeysCount = Object.keys(expectedMapValue).length; + + expect(root.get(mapId).size()).to.equal( + expectedKeysCount, + `Check map #${i + 1} has expected number of keys after MAP_CREATE ops`, + ); + Object.entries(expectedMapValue).forEach(([key, value]) => { + expect(root.get(mapId).get(key)).to.equal( + value, + `Check map #${i + 1} has expected value for "${key}" key after MAP_CREATE ops`, + ); + }); + } + }, + }, + + { + allTransportsAndProtocols: true, + description: 'can apply MAP_SET with primitives object operation messages', + action: async (ctx) => { + const { root, objectsHelper, channelName, helper } = ctx; + + // check root is empty before ops + primitiveKeyData.forEach((keyData) => { + expect( + root.get(keyData.key), + `Check "${keyData.key}" key doesn't exist on root before applying MAP_SET ops`, + ).to.not.exist; + }); + + const keysUpdatedPromise = Promise.all(primitiveKeyData.map((x) => waitForMapKeyUpdate(root, x.key))); + // apply MAP_SET ops + await Promise.all( + primitiveKeyData.map((keyData) => + objectsHelper.operationRequest( + channelName, + objectsHelper.mapSetRestOp({ + objectId: 'root', + key: keyData.key, + value: keyData.data, + }), + ), + ), + ); + await keysUpdatedPromise; + + // check everything is applied correctly + primitiveKeyData.forEach((keyData) => { + if (keyData.data.bytes != null) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.bytes)), + `Check root has correct value for "${keyData.key}" key after MAP_SET op`, + ).to.be.true; + } else { + const valueType = typeof root.get(keyData.key); + expect(root.get(keyData.key)).to.equal( + keyData.data[valueType], + `Check root has correct value for "${keyData.key}" key after MAP_SET op`, + ); + } + }); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'can apply MAP_SET with object ids object operation messages', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + + // check no object ids are set on root + expect( + root.get('keyToCounter'), + `Check "keyToCounter" key doesn't exist on root before applying MAP_SET ops`, + ).to.not.exist; + expect(root.get('keyToMap'), `Check "keyToMap" key doesn't exist on root before applying MAP_SET ops`).to + .not.exist; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'keyToCounter'), + waitForMapKeyUpdate(root, 'keyToMap'), + ]); + // create new objects and set on root + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'keyToCounter', + createOp: objectsHelper.counterCreateRestOp({ number: 1 }), + }); + + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'keyToMap', + createOp: objectsHelper.mapCreateRestOp({ + data: { + stringKey: { string: 'stringValue' }, + }, + }), + }); + await objectsCreatedPromise; + + // check root has refs to new objects and they are not zero-value + const counter = root.get('keyToCounter'); + const map = root.get('keyToMap'); + + expect(counter, 'Check counter at "keyToCounter" key in root exists').to.exist; + expectInstanceOf( + counter, + 'LiveCounter', + 'Check counter at "keyToCounter" key in root is of type LiveCounter', + ); + expect(counter.value()).to.equal(1, 'Check counter at "keyToCounter" key in root has correct value'); + + expect(map, 'Check map at "keyToMap" key in root exists').to.exist; + expectInstanceOf(map, 'LiveMap', 'Check map at "keyToMap" key in root is of type LiveMap'); + expect(map.size()).to.equal(1, 'Check map at "keyToMap" key in root has correct number of keys'); + expect(map.get('stringKey')).to.equal( + 'stringValue', + 'Check map at "keyToMap" key in root has correct "stringKey" value', + ); + }, + }, + + { + description: + 'MAP_SET object operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, objectsHelper, channel } = ctx; + + // create new map and set it on a root with forged timeserials + const mapId = objectsHelper.fakeMapObjectId(); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', + state: [ + objectsHelper.mapCreateOp({ + objectId: mapId, + entries: { + foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo3: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo5: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo6: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + }, + }), + ], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'map', data: { objectId: mapId } })], + }); + + // inject operations with various timeserial values + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // existing site, earlier site CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, same site CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, later site CGO, applied, site timeserials updated + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, same site CGO (updated from last op), not applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier entry CGO, not applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later entry CGO, applied + ].entries()) { + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial, + siteCode, + state: [objectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { string: 'baz' } })], + }); + } + + // check only operations with correct timeserials were applied + const expectedMapKeys = [ + { key: 'foo1', value: 'bar' }, + { key: 'foo2', value: 'bar' }, + { key: 'foo3', value: 'baz' }, // updated + { key: 'foo4', value: 'bar' }, + { key: 'foo5', value: 'bar' }, + { key: 'foo6', value: 'baz' }, // updated + ]; + + expectedMapKeys.forEach(({ key, value }) => { + expect(root.get('map').get(key)).to.equal( + value, + `Check "${key}" key on map has expected value after MAP_SET ops`, + ); + }); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'can apply MAP_REMOVE object operation messages', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + const mapKey = 'map'; + + const mapCreatedPromise = waitForMapKeyUpdate(root, mapKey); + // create new map and set on root + const { objectId: mapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: mapKey, + createOp: objectsHelper.mapCreateRestOp({ + data: { + shouldStay: { string: 'foo' }, + shouldDelete: { string: 'bar' }, + }, + }), + }); + await mapCreatedPromise; + + const map = root.get(mapKey); + // check map has expected keys before MAP_REMOVE ops + expect(map.size()).to.equal( + 2, + `Check map at "${mapKey}" key in root has correct number of keys before MAP_REMOVE`, + ); + expect(map.get('shouldStay')).to.equal( + 'foo', + `Check map at "${mapKey}" key in root has correct "shouldStay" value before MAP_REMOVE`, + ); + expect(map.get('shouldDelete')).to.equal( + 'bar', + `Check map at "${mapKey}" key in root has correct "shouldDelete" value before MAP_REMOVE`, + ); + + const keyRemovedPromise = waitForMapKeyUpdate(map, 'shouldDelete'); + // send MAP_REMOVE op + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapRemoveRestOp({ + objectId: mapObjectId, + key: 'shouldDelete', + }), + ); + await keyRemovedPromise; + + // check map has correct keys after MAP_REMOVE ops + expect(map.size()).to.equal( + 1, + `Check map at "${mapKey}" key in root has correct number of keys after MAP_REMOVE`, + ); + expect(map.get('shouldStay')).to.equal( + 'foo', + `Check map at "${mapKey}" key in root has correct "shouldStay" value after MAP_REMOVE`, + ); + expect( + map.get('shouldDelete'), + `Check map at "${mapKey}" key in root has no "shouldDelete" key after MAP_REMOVE`, + ).to.not.exist; + }, + }, + + { + description: + 'MAP_REMOVE object operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, objectsHelper, channel } = ctx; + + // create new map and set it on a root with forged timeserials + const mapId = objectsHelper.fakeMapObjectId(); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', + state: [ + objectsHelper.mapCreateOp({ + objectId: mapId, + entries: { + foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo3: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo5: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo6: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + }, + }), + ], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'map', data: { objectId: mapId } })], + }); + + // inject operations with various timeserial values + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // existing site, earlier site CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, same site CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, later site CGO, applied, site timeserials updated + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, same site CGO (updated from last op), not applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier entry CGO, not applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later entry CGO, applied + ].entries()) { + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial, + siteCode, + state: [objectsHelper.mapRemoveOp({ objectId: mapId, key: `foo${i + 1}` })], + }); + } + + // check only operations with correct timeserials were applied + const expectedMapKeys = [ + { key: 'foo1', exists: true }, + { key: 'foo2', exists: true }, + { key: 'foo3', exists: false }, // removed + { key: 'foo4', exists: true }, + { key: 'foo5', exists: true }, + { key: 'foo6', exists: false }, // removed + ]; + + expectedMapKeys.forEach(({ key, exists }) => { + if (exists) { + expect(root.get('map').get(key), `Check "${key}" key on map still exists after MAP_REMOVE ops`).to + .exist; + } else { + expect(root.get('map').get(key), `Check "${key}" key on map does not exist after MAP_REMOVE ops`).to.not + .exist; + } + }); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'can apply COUNTER_CREATE object operation messages', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + + // Objects public API allows us to check value of objects we've created based on COUNTER_CREATE ops + // if we assign those objects to another map (root for example), as there is no way to access those objects from the internal pool directly. + // however, in this test we put heavy focus on the data that is being created as the result of the COUNTER_CREATE op. + + // check no counters exist on root + countersFixtures.forEach((fixture) => { + const key = fixture.name; + expect(root.get(key), `Check "${key}" key doesn't exist on root before applying COUNTER_CREATE ops`).to + .not.exist; + }); + + const countersCreatedPromise = Promise.all(countersFixtures.map((x) => waitForMapKeyUpdate(root, x.name))); + // create new counters and set on root + await Promise.all( + countersFixtures.map((fixture) => + objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: fixture.name, + createOp: objectsHelper.counterCreateRestOp({ number: fixture.count }), + }), + ), + ); + await countersCreatedPromise; + + // check created counters + countersFixtures.forEach((fixture) => { + const key = fixture.name; + const counterObj = root.get(key); + + // check all counters exist on root + expect(counterObj, `Check counter at "${key}" key in root exists`).to.exist; + expectInstanceOf( + counterObj, + 'LiveCounter', + `Check counter at "${key}" key in root is of type LiveCounter`, + ); + + // check counters have correct values + expect(counterObj.value()).to.equal( + // if count was not set, should default to 0 + fixture.count ?? 0, + `Check counter at "${key}" key in root has correct value`, + ); + }); + }, + }, + + { + description: + 'COUNTER_CREATE object operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, objectsHelper, channel } = ctx; + + // need to use multiple counters as COUNTER_CREATE op can only be applied once to a counter object + const counterIds = [ + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + ]; + await Promise.all( + counterIds.map(async (counterId, i) => { + // send a COUNTER_INC op first to create a zero-value counter with forged site timeserials vector (from the op), and set it on a root. + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', + state: [objectsHelper.counterIncOp({ objectId: counterId, amount: 1 })], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: counterId, data: { objectId: counterId } })], + }); + }), + ); + + // inject operations with various timeserial values + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // existing site, earlier CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, same CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, later CGO, applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied + ].entries()) { + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial, + siteCode, + state: [objectsHelper.counterCreateOp({ objectId: counterIds[i], count: 10 })], + }); + } + + // check only operations with correct timeserials were applied + const expectedCounterValues = [ + 1, + 1, + 11, // applied COUNTER_CREATE + 11, // applied COUNTER_CREATE + 11, // applied COUNTER_CREATE + ]; + + for (const [i, counterId] of counterIds.entries()) { + const expectedValue = expectedCounterValues[i]; + + expect(root.get(counterId).value()).to.equal( + expectedValue, + `Check counter #${i + 1} has expected value after COUNTER_CREATE ops`, + ); + } + }, + }, + + { + allTransportsAndProtocols: true, + description: 'can apply COUNTER_INC object operation messages', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + const counterKey = 'counter'; + let expectedCounterValue = 0; + + const counterCreated = waitForMapKeyUpdate(root, counterKey); + // create new counter and set on root + const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: counterKey, + createOp: objectsHelper.counterCreateRestOp({ number: expectedCounterValue }), + }); + await counterCreated; + + const counter = root.get(counterKey); + // check counter has expected value before COUNTER_INC + expect(counter.value()).to.equal( + expectedCounterValue, + `Check counter at "${counterKey}" key in root has correct value before COUNTER_INC`, + ); + + const increments = [ + 1, // value=1 + 10, // value=11 + 100, // value=111 + 1000000, // value=1000111 + -1000111, // value=0 + -1, // value=-1 + -10, // value=-11 + -100, // value=-111 + -1000000, // value=-1000111 + 1000111, // value=0 + Number.MAX_SAFE_INTEGER, // value=9007199254740991 + // do next decrements in 2 steps as opposed to multiplying by -2 to prevent overflow + -Number.MAX_SAFE_INTEGER, // value=0 + -Number.MAX_SAFE_INTEGER, // value=-9007199254740991 + ]; + + // send increments one at a time and check expected value + for (let i = 0; i < increments.length; i++) { + const increment = increments[i]; + expectedCounterValue += increment; + + const counterUpdatedPromise = waitForCounterUpdate(counter); + await objectsHelper.operationRequest( + channelName, + objectsHelper.counterIncRestOp({ + objectId: counterObjectId, + number: increment, + }), + ); + await counterUpdatedPromise; + + expect(counter.value()).to.equal( + expectedCounterValue, + `Check counter at "${counterKey}" key in root has correct value after ${i + 1} COUNTER_INC ops`, + ); + } + }, + }, + + { + description: + 'COUNTER_INC object operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, objectsHelper, channel } = ctx; + + // create new counter and set it on a root with forged timeserials + const counterId = objectsHelper.fakeCounterObjectId(); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', + state: [objectsHelper.counterCreateOp({ objectId: counterId, count: 1 })], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'counter', data: { objectId: counterId } })], + }); + + // inject operations with various timeserial values + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // +10 existing site, earlier CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // +100 existing site, same CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // +1000 existing site, later CGO, applied, site timeserials updated + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // +10000 existing site, same CGO (updated from last op), not applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // +100000 different site, earlier CGO, applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // +1000000 different site, later CGO, applied + ].entries()) { + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial, + siteCode, + state: [objectsHelper.counterIncOp({ objectId: counterId, amount: Math.pow(10, i + 1) })], + }); + } + + // check only operations with correct timeserials were applied + expect(root.get('counter').value()).to.equal( + 1 + 1000 + 100000 + 1000000, // sum of passing operations and the initial value + `Check counter has expected value after COUNTER_INC ops`, + ); + }, + }, + + { + description: 'can apply OBJECT_DELETE object operation messages', + action: async (ctx) => { + const { root, objectsHelper, channelName, channel } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'map'), + waitForMapKeyUpdate(root, 'counter'), + ]); + // create initial objects and set on root + const { objectId: mapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: objectsHelper.mapCreateRestOp(), + }); + const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: objectsHelper.counterCreateRestOp(), + }); + await objectsCreatedPromise; + + expect(root.get('map'), 'Check map exists on root before OBJECT_DELETE').to.exist; + expect(root.get('counter'), 'Check counter exists on root before OBJECT_DELETE').to.exist; + + // inject OBJECT_DELETE + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.objectDeleteOp({ objectId: mapObjectId })], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 0), + siteCode: 'aaa', + state: [objectsHelper.objectDeleteOp({ objectId: counterObjectId })], + }); + + expect(root.get('map'), 'Check map is not accessible on root after OBJECT_DELETE').to.not.exist; + expect(root.get('counter'), 'Check counter is not accessible on root after OBJECT_DELETE').to.not.exist; + }, + }, + + { + description: 'OBJECT_DELETE for unknown object id creates zero-value tombstoned object', + action: async (ctx) => { + const { root, objectsHelper, channel } = ctx; + + const counterId = objectsHelper.fakeCounterObjectId(); + // inject OBJECT_DELETE. should create a zero-value tombstoned object which can't be modified + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.objectDeleteOp({ objectId: counterId })], + }); + + // try to create and set tombstoned object on root + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 0, 0), + siteCode: 'bbb', + state: [objectsHelper.counterCreateOp({ objectId: counterId })], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'counter', data: { objectId: counterId } })], + }); + + expect(root.get('counter'), 'Check counter is not accessible on root').to.not.exist; + }, + }, + + { + description: + 'OBJECT_DELETE object operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, objectsHelper, channel } = ctx; + + // need to use multiple objects as OBJECT_DELETE op can only be applied once to an object + const counterIds = [ + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + objectsHelper.fakeCounterObjectId(), + ]; + await Promise.all( + counterIds.map(async (counterId, i) => { + // create objects and set them on root with forged timeserials + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 1, 0), + siteCode: 'bbb', + state: [objectsHelper.counterCreateOp({ objectId: counterId })], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: counterId, data: { objectId: counterId } })], + }); + }), + ); + + // inject OBJECT_DELETE operations with various timeserial values + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // existing site, earlier CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, same CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, later CGO, applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // different site, earlier CGO, applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // different site, later CGO, applied + ].entries()) { + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial, + siteCode, + state: [objectsHelper.objectDeleteOp({ objectId: counterIds[i] })], + }); + } + + // check only operations with correct timeserials were applied + const expectedCounters = [ + { exists: true }, + { exists: true }, + { exists: false }, // OBJECT_DELETE applied + { exists: false }, // OBJECT_DELETE applied + { exists: false }, // OBJECT_DELETE applied + ]; + + for (const [i, counterId] of counterIds.entries()) { + const { exists } = expectedCounters[i]; + + if (exists) { + expect( + root.get(counterId), + `Check counter #${i + 1} exists on root as OBJECT_DELETE op was not applied`, + ).to.exist; + } else { + expect( + root.get(counterId), + `Check counter #${i + 1} does not exist on root as OBJECT_DELETE op was applied`, + ).to.not.exist; + } + } + }, + }, + + { + description: 'OBJECT_DELETE triggers subscription callback with deleted data', + action: async (ctx) => { + const { root, objectsHelper, channelName, channel } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'map'), + waitForMapKeyUpdate(root, 'counter'), + ]); + // create initial objects and set on root + const { objectId: mapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: objectsHelper.mapCreateRestOp({ + data: { + foo: { string: 'bar' }, + baz: { number: 1 }, + }, + }), + }); + const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: objectsHelper.counterCreateRestOp({ number: 1 }), + }); + await objectsCreatedPromise; + + const mapSubPromise = new Promise((resolve, reject) => + root.get('map').subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { foo: 'removed', baz: 'removed' } }, + 'Check map subscription callback is called with an expected update object after OBJECT_DELETE operation', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + const counterSubPromise = new Promise((resolve, reject) => + root.get('counter').subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { amount: -1 } }, + 'Check counter subscription callback is called with an expected update object after OBJECT_DELETE operation', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + + // inject OBJECT_DELETE + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.objectDeleteOp({ objectId: mapObjectId })], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 0), + siteCode: 'aaa', + state: [objectsHelper.objectDeleteOp({ objectId: counterObjectId })], + }); + + await Promise.all([mapSubPromise, counterSubPromise]); + }, + }, + + { + description: 'MAP_SET with reference to a tombstoned object results in undefined value on key', + action: async (ctx) => { + const { root, objectsHelper, channelName, channel } = ctx; + + const objectCreatedPromise = waitForMapKeyUpdate(root, 'foo'); + // create initial objects and set on root + const { objectId: counterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'foo', + createOp: objectsHelper.counterCreateRestOp(), + }); + await objectCreatedPromise; + + expect(root.get('foo'), 'Check counter exists on root before OBJECT_DELETE').to.exist; + + // inject OBJECT_DELETE + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.objectDeleteOp({ objectId: counterObjectId })], + }); + + // set tombstoned counter to another key on root + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'bar', data: { objectId: counterObjectId } })], + }); + + expect(root.get('bar'), 'Check counter is not accessible on new key in root after OBJECT_DELETE').to.not + .exist; + }, + }, + + { + description: 'object operation message on a tombstoned object does not revive it', + action: async (ctx) => { + const { root, objectsHelper, channelName, channel } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'map1'), + waitForMapKeyUpdate(root, 'map2'), + waitForMapKeyUpdate(root, 'counter1'), + ]); + // create initial objects and set on root + const { objectId: mapId1 } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map1', + createOp: objectsHelper.mapCreateRestOp(), + }); + const { objectId: mapId2 } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map2', + createOp: objectsHelper.mapCreateRestOp({ data: { foo: { string: 'bar' } } }), + }); + const { objectId: counterId1 } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter1', + createOp: objectsHelper.counterCreateRestOp(), + }); + await objectsCreatedPromise; + + expect(root.get('map1'), 'Check map1 exists on root before OBJECT_DELETE').to.exist; + expect(root.get('map2'), 'Check map2 exists on root before OBJECT_DELETE').to.exist; + expect(root.get('counter1'), 'Check counter1 exists on root before OBJECT_DELETE').to.exist; + + // inject OBJECT_DELETE + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.objectDeleteOp({ objectId: mapId1 })], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 0), + siteCode: 'aaa', + state: [objectsHelper.objectDeleteOp({ objectId: mapId2 })], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 2, 0), + siteCode: 'aaa', + state: [objectsHelper.objectDeleteOp({ objectId: counterId1 })], + }); + + // inject object operations on tombstoned objects + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 3, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: mapId1, key: 'baz', data: { string: 'qux' } })], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 4, 0), + siteCode: 'aaa', + state: [objectsHelper.mapRemoveOp({ objectId: mapId2, key: 'foo' })], + }); + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 5, 0), + siteCode: 'aaa', + state: [objectsHelper.counterIncOp({ objectId: counterId1, amount: 1 })], + }); + + // objects should still be deleted + expect(root.get('map1'), 'Check map1 does not exist on root after OBJECT_DELETE and another object op').to + .not.exist; + expect(root.get('map2'), 'Check map2 does not exist on root after OBJECT_DELETE and another object op').to + .not.exist; + expect( + root.get('counter1'), + 'Check counter1 does not exist on root after OBJECT_DELETE and another object op', + ).to.not.exist; + }, + }, + ]; + + const applyOperationsDuringSyncScenarios = [ + { + description: 'object operation messages are buffered during OBJECT_SYNC sequence', + action: async (ctx) => { + const { root, objectsHelper, channel, client, helper } = ctx; + + // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:cursor', + }); + + // inject operations, it should not be applied as sync is in progress + await Promise.all( + primitiveKeyData.map(async (keyData) => { + // copy data object as library will modify it + const data = { ...keyData.data }; + helper.recordPrivateApi('read.realtime.options.useBinaryProtocol'); + if (data.bytes != null && client.options.useBinaryProtocol) { + // decode base64 data to binary for binary protocol + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + data.bytes = BufferUtils.base64Decode(data.bytes); + } + + return objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data })], + }); + }), + ); + + // check root doesn't have data from operations + primitiveKeyData.forEach((keyData) => { + expect(root.get(keyData.key), `Check "${keyData.key}" key doesn't exist on root during OBJECT_SYNC`).to + .not.exist; + }); + }, + }, + + { + description: 'buffered object operation messages are applied when OBJECT_SYNC sequence ends', + action: async (ctx) => { + const { root, objectsHelper, channel, helper, client } = ctx; + + // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:cursor', + }); + + // inject operations, they should be applied when sync ends + await Promise.all( + primitiveKeyData.map(async (keyData, i) => { + // copy data object as library will modify it + const data = { ...keyData.data }; + helper.recordPrivateApi('read.realtime.options.useBinaryProtocol'); + if (data.bytes != null && client.options.useBinaryProtocol) { + // decode base64 data to binary for binary protocol + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + data.bytes = BufferUtils.base64Decode(data.bytes); + } + + return objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data })], + }); + }), + ); + + // end the sync with empty cursor + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:', + }); + + // check everything is applied correctly + primitiveKeyData.forEach((keyData) => { + if (keyData.data.bytes != null) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.bytes)), + `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, + ).to.be.true; + } else { + const valueType = typeof root.get(keyData.key); + expect(root.get(keyData.key)).to.equal( + keyData.data[valueType], + `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, + ); + } + }); + }, + }, + + { + description: 'buffered object operation messages are discarded when new OBJECT_SYNC sequence starts', + action: async (ctx) => { + const { root, objectsHelper, channel, client, helper } = ctx; + + // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:cursor', + }); + + // inject operations, expect them to be discarded when sync with new sequence id starts + await Promise.all( + primitiveKeyData.map(async (keyData, i) => { + // copy data object as library will modify it + const data = { ...keyData.data }; + helper.recordPrivateApi('read.realtime.options.useBinaryProtocol'); + if (data.bytes != null && client.options.useBinaryProtocol) { + // decode base64 data to binary for binary protocol + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + data.bytes = BufferUtils.base64Decode(data.bytes); + } + + return objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data })], + }); + }), + ); + + // start new sync with new sequence id + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'otherserial:cursor', + }); + + // inject another operation that should be applied when latest sync ends + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('bbb', 0, 0), + siteCode: 'bbb', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: 'foo', data: { string: 'bar' } })], + }); + + // end sync + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'otherserial:', + }); + + // check root doesn't have data from operations received during first sync + primitiveKeyData.forEach((keyData) => { + expect( + root.get(keyData.key), + `Check "${keyData.key}" key doesn't exist on root when OBJECT_SYNC has ended`, + ).to.not.exist; + }); + + // check root has data from operations received during second sync + expect(root.get('foo')).to.equal( + 'bar', + 'Check root has data from operations received during second OBJECT_SYNC sequence', + ); + }, + }, + + { + description: + 'buffered object operation messages are applied based on the site timeserials vector of the object', + action: async (ctx) => { + const { root, objectsHelper, channel } = ctx; + + // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages + const mapId = objectsHelper.fakeMapObjectId(); + const counterId = objectsHelper.fakeCounterObjectId(); + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:cursor', + // add object state messages with non-empty site timeserials + state: [ + // next map and counter objects will be checked to have correct operations applied on them based on site timeserials + objectsHelper.mapObject({ + objectId: mapId, + siteTimeserials: { + bbb: lexicoTimeserial('bbb', 2, 0), + ccc: lexicoTimeserial('ccc', 5, 0), + }, + materialisedEntries: { + foo1: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo2: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo3: { timeserial: lexicoTimeserial('ccc', 5, 0), data: { string: 'bar' } }, + foo4: { timeserial: lexicoTimeserial('bbb', 0, 0), data: { string: 'bar' } }, + foo5: { timeserial: lexicoTimeserial('bbb', 2, 0), data: { string: 'bar' } }, + foo6: { timeserial: lexicoTimeserial('ccc', 2, 0), data: { string: 'bar' } }, + foo7: { timeserial: lexicoTimeserial('ccc', 0, 0), data: { string: 'bar' } }, + foo8: { timeserial: lexicoTimeserial('ccc', 0, 0), data: { string: 'bar' } }, + }, + }), + objectsHelper.counterObject({ + objectId: counterId, + siteTimeserials: { + bbb: lexicoTimeserial('bbb', 1, 0), + }, + initialCount: 1, + }), + // add objects to the root so they're discoverable in the object tree + objectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + initialEntries: { + map: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: mapId } }, + counter: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId } }, + }, + }), + ], + }); + + // inject operations with various timeserial values + // Map: + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // existing site, earlier site CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // existing site, same site CGO, not applied + { serial: lexicoTimeserial('bbb', 3, 0), siteCode: 'bbb' }, // existing site, later site CGO, earlier entry CGO, not applied but site timeserial updated + // message with later site CGO, same entry CGO case is not possible, as timeserial from entry would be set for the corresponding site code or be less than that + { serial: lexicoTimeserial('bbb', 3, 0), siteCode: 'bbb' }, // existing site, same site CGO (updated from last op), later entry CGO, not applied + { serial: lexicoTimeserial('bbb', 4, 0), siteCode: 'bbb' }, // existing site, later site CGO, later entry CGO, applied + { serial: lexicoTimeserial('aaa', 1, 0), siteCode: 'aaa' }, // different site, earlier entry CGO, not applied but site timeserial updated + { serial: lexicoTimeserial('aaa', 1, 0), siteCode: 'aaa' }, // different site, same site CGO (updated from last op), later entry CGO, not applied + // different site with matching entry CGO case is not possible, as matching entry timeserial means that that timeserial is in the site timeserials vector + { serial: lexicoTimeserial('ddd', 1, 0), siteCode: 'ddd' }, // different site, later entry CGO, applied + ].entries()) { + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial, + siteCode, + state: [objectsHelper.mapSetOp({ objectId: mapId, key: `foo${i + 1}`, data: { string: 'baz' } })], + }); + } + + // Counter: + for (const [i, { serial, siteCode }] of [ + { serial: lexicoTimeserial('bbb', 0, 0), siteCode: 'bbb' }, // +10 existing site, earlier CGO, not applied + { serial: lexicoTimeserial('bbb', 1, 0), siteCode: 'bbb' }, // +100 existing site, same CGO, not applied + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // +1000 existing site, later CGO, applied, site timeserials updated + { serial: lexicoTimeserial('bbb', 2, 0), siteCode: 'bbb' }, // +10000 existing site, same CGO (updated from last op), not applied + { serial: lexicoTimeserial('aaa', 0, 0), siteCode: 'aaa' }, // +100000 different site, earlier CGO, applied + { serial: lexicoTimeserial('ccc', 9, 0), siteCode: 'ccc' }, // +1000000 different site, later CGO, applied + ].entries()) { + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial, + siteCode, + state: [objectsHelper.counterIncOp({ objectId: counterId, amount: Math.pow(10, i + 1) })], + }); + } + + // end sync + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:', + }); + + // check only operations with correct timeserials were applied + const expectedMapKeys = [ + { key: 'foo1', value: 'bar' }, + { key: 'foo2', value: 'bar' }, + { key: 'foo3', value: 'bar' }, + { key: 'foo4', value: 'bar' }, + { key: 'foo5', value: 'baz' }, // updated + { key: 'foo6', value: 'bar' }, + { key: 'foo7', value: 'bar' }, + { key: 'foo8', value: 'baz' }, // updated + ]; + + expectedMapKeys.forEach(({ key, value }) => { + expect(root.get('map').get(key)).to.equal( + value, + `Check "${key}" key on map has expected value after OBJECT_SYNC has ended`, + ); + }); + + expect(root.get('counter').value()).to.equal( + 1 + 1000 + 100000 + 1000000, // sum of passing operations and the initial value + `Check counter has expected value after OBJECT_SYNC has ended`, + ); + }, + }, + + { + description: + 'subsequent object operation messages are applied immediately after OBJECT_SYNC ended and buffers are applied', + action: async (ctx) => { + const { root, objectsHelper, channel, channelName, helper, client } = ctx; + + // start new sync sequence with a cursor so client will wait for the next OBJECT_SYNC messages + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:cursor', + }); + + // inject operations, they should be applied when sync ends + await Promise.all( + primitiveKeyData.map(async (keyData, i) => { + // copy data object as library will modify it + const data = { ...keyData.data }; + helper.recordPrivateApi('read.realtime.options.useBinaryProtocol'); + if (data.bytes != null && client.options.useBinaryProtocol) { + // decode base64 data to binary for binary protocol + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + data.bytes = BufferUtils.base64Decode(data.bytes); + } + + return objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', i, 0), + siteCode: 'aaa', + state: [objectsHelper.mapSetOp({ objectId: 'root', key: keyData.key, data })], + }); + }), + ); + + // end the sync with empty cursor + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:', + }); + + const keyUpdatedPromise = waitForMapKeyUpdate(root, 'foo'); + // send some more operations + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapSetRestOp({ + objectId: 'root', + key: 'foo', + value: { string: 'bar' }, + }), + ); + await keyUpdatedPromise; + + // check buffered operations are applied, as well as the most recent operation outside of the sync sequence is applied + primitiveKeyData.forEach((keyData) => { + if (keyData.data.bytes != null) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.bytes)), + `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, + ).to.be.true; + } else { + const valueType = typeof root.get(keyData.key); + expect(root.get(keyData.key)).to.equal( + keyData.data[valueType], + `Check root has correct value for "${keyData.key}" key after OBJECT_SYNC has ended and buffered operations are applied`, + ); + } + }); + expect(root.get('foo')).to.equal( + 'bar', + 'Check root has correct value for "foo" key from operation received outside of OBJECT_SYNC after other buffered operations were applied', + ); + }, + }, + ]; + + const writeApiScenarios = [ + { + allTransportsAndProtocols: true, + description: 'LiveCounter.increment sends COUNTER_INC operation', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: objectsHelper.counterCreateRestOp(), + }); + await counterCreatedPromise; + + const counter = root.get('counter'); + const increments = [ + 1, // value=1 + 10, // value=11 + -11, // value=0 + -1, // value=-1 + -10, // value=-11 + 11, // value=0 + Number.MAX_SAFE_INTEGER, // value=9007199254740991 + -Number.MAX_SAFE_INTEGER, // value=0 + -Number.MAX_SAFE_INTEGER, // value=-9007199254740991 + ]; + let expectedCounterValue = 0; + + for (let i = 0; i < increments.length; i++) { + const increment = increments[i]; + expectedCounterValue += increment; + + const counterUpdatedPromise = waitForCounterUpdate(counter); + await counter.increment(increment); + await counterUpdatedPromise; + + expect(counter.value()).to.equal( + expectedCounterValue, + `Check counter has correct value after ${i + 1} LiveCounter.increment calls`, + ); + } + }, + }, + + { + description: 'LiveCounter.increment throws on invalid input', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: objectsHelper.counterCreateRestOp(), + }); + await counterCreatedPromise; + + const counter = root.get('counter'); + + await expectToThrowAsync( + async () => counter.increment(), + 'Counter value increment should be a valid number', + ); + await expectToThrowAsync( + async () => counter.increment(null), + 'Counter value increment should be a valid number', + ); + await expectToThrowAsync( + async () => counter.increment(Number.NaN), + 'Counter value increment should be a valid number', + ); + await expectToThrowAsync( + async () => counter.increment(Number.POSITIVE_INFINITY), + 'Counter value increment should be a valid number', + ); + await expectToThrowAsync( + async () => counter.increment(Number.NEGATIVE_INFINITY), + 'Counter value increment should be a valid number', + ); + await expectToThrowAsync( + async () => counter.increment('foo'), + 'Counter value increment should be a valid number', + ); + await expectToThrowAsync( + async () => counter.increment(BigInt(1)), + 'Counter value increment should be a valid number', + ); + await expectToThrowAsync( + async () => counter.increment(true), + 'Counter value increment should be a valid number', + ); + await expectToThrowAsync( + async () => counter.increment(Symbol()), + 'Counter value increment should be a valid number', + ); + await expectToThrowAsync( + async () => counter.increment({}), + 'Counter value increment should be a valid number', + ); + await expectToThrowAsync( + async () => counter.increment([]), + 'Counter value increment should be a valid number', + ); + await expectToThrowAsync( + async () => counter.increment(counter), + 'Counter value increment should be a valid number', + ); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveCounter.decrement sends COUNTER_INC operation', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: objectsHelper.counterCreateRestOp(), + }); + await counterCreatedPromise; + + const counter = root.get('counter'); + const decrements = [ + 1, // value=-1 + 10, // value=-11 + -11, // value=0 + -1, // value=1 + -10, // value=11 + 11, // value=0 + Number.MAX_SAFE_INTEGER, // value=-9007199254740991 + -Number.MAX_SAFE_INTEGER, // value=0 + -Number.MAX_SAFE_INTEGER, // value=9007199254740991 + ]; + let expectedCounterValue = 0; + + for (let i = 0; i < decrements.length; i++) { + const decrement = decrements[i]; + expectedCounterValue -= decrement; + + const counterUpdatedPromise = waitForCounterUpdate(counter); + await counter.decrement(decrement); + await counterUpdatedPromise; + + expect(counter.value()).to.equal( + expectedCounterValue, + `Check counter has correct value after ${i + 1} LiveCounter.decrement calls`, + ); + } + }, + }, + + { + description: 'LiveCounter.decrement throws on invalid input', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: objectsHelper.counterCreateRestOp(), + }); + await counterCreatedPromise; + + const counter = root.get('counter'); + + await expectToThrowAsync( + async () => counter.decrement(), + 'Counter value decrement should be a valid number', + ); + await expectToThrowAsync( + async () => counter.decrement(null), + 'Counter value decrement should be a valid number', + ); + await expectToThrowAsync( + async () => counter.decrement(Number.NaN), + 'Counter value decrement should be a valid number', + ); + await expectToThrowAsync( + async () => counter.decrement(Number.POSITIVE_INFINITY), + 'Counter value decrement should be a valid number', + ); + await expectToThrowAsync( + async () => counter.decrement(Number.NEGATIVE_INFINITY), + 'Counter value decrement should be a valid number', + ); + await expectToThrowAsync( + async () => counter.decrement('foo'), + 'Counter value decrement should be a valid number', + ); + await expectToThrowAsync( + async () => counter.decrement(BigInt(1)), + 'Counter value decrement should be a valid number', + ); + await expectToThrowAsync( + async () => counter.decrement(true), + 'Counter value decrement should be a valid number', + ); + await expectToThrowAsync( + async () => counter.decrement(Symbol()), + 'Counter value decrement should be a valid number', + ); + await expectToThrowAsync( + async () => counter.decrement({}), + 'Counter value decrement should be a valid number', + ); + await expectToThrowAsync( + async () => counter.decrement([]), + 'Counter value decrement should be a valid number', + ); + await expectToThrowAsync( + async () => counter.decrement(counter), + 'Counter value decrement should be a valid number', + ); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveMap.set sends MAP_SET operation with primitive values', + action: async (ctx) => { + const { root, helper } = ctx; + + const keysUpdatedPromise = Promise.all(primitiveKeyData.map((x) => waitForMapKeyUpdate(root, x.key))); + await Promise.all( + primitiveKeyData.map(async (keyData) => { + let value; + if (keyData.data.bytes != null) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + value = BufferUtils.base64Decode(keyData.data.bytes); + } else if (keyData.data.number != null) { + value = keyData.data.number; + } else if (keyData.data.string != null) { + value = keyData.data.string; + } else if (keyData.data.boolean != null) { + value = keyData.data.boolean; + } + + await root.set(keyData.key, value); + }), + ); + await keysUpdatedPromise; + + // check everything is applied correctly + primitiveKeyData.forEach((keyData) => { + if (keyData.data.bytes != null) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual(root.get(keyData.key), BufferUtils.base64Decode(keyData.data.bytes)), + `Check root has correct value for "${keyData.key}" key after LiveMap.set call`, + ).to.be.true; + } else { + const valueType = typeof root.get(keyData.key); + expect(root.get(keyData.key)).to.equal( + keyData.data[valueType], + `Check root has correct value for "${keyData.key}" key after LiveMap.set call`, + ); + } + }); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveMap.set sends MAP_SET operation with reference to another LiveObject', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: objectsHelper.counterCreateRestOp(), + }); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: objectsHelper.mapCreateRestOp(), + }); + await objectsCreatedPromise; + + const counter = root.get('counter'); + const map = root.get('map'); + + const keysUpdatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter2'), + waitForMapKeyUpdate(root, 'map2'), + ]); + await root.set('counter2', counter); + await root.set('map2', map); + await keysUpdatedPromise; + + expect(root.get('counter2')).to.equal( + counter, + 'Check can set a reference to a LiveCounter object on a root via a LiveMap.set call', + ); + expect(root.get('map2')).to.equal( + map, + 'Check can set a reference to a LiveMap object on a root via a LiveMap.set call', + ); + }, + }, + + { + description: 'LiveMap.set throws on invalid input', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + + const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: objectsHelper.mapCreateRestOp(), + }); + await mapCreatedPromise; + + const map = root.get('map'); + + await expectToThrowAsync(async () => map.set(), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(null), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(1), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(BigInt(1)), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(true), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(Symbol()), 'Map key should be string'); + await expectToThrowAsync(async () => map.set({}), 'Map key should be string'); + await expectToThrowAsync(async () => map.set([]), 'Map key should be string'); + await expectToThrowAsync(async () => map.set(map), 'Map key should be string'); + + await expectToThrowAsync(async () => map.set('key'), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => map.set('key', null), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => map.set('key', BigInt(1)), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => map.set('key', Symbol()), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => map.set('key', {}), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => map.set('key', []), 'Map value data type is unsupported'); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveMap.remove sends MAP_REMOVE operation', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + + const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: objectsHelper.mapCreateRestOp({ + data: { + foo: { number: 1 }, + bar: { number: 1 }, + baz: { number: 1 }, + }, + }), + }); + await mapCreatedPromise; + + const map = root.get('map'); + + const keysUpdatedPromise = Promise.all([waitForMapKeyUpdate(map, 'foo'), waitForMapKeyUpdate(map, 'bar')]); + await map.remove('foo'); + await map.remove('bar'); + await keysUpdatedPromise; + + expect(map.get('foo'), 'Check can remove a key from a root via a LiveMap.remove call').to.not.exist; + expect(map.get('bar'), 'Check can remove a key from a root via a LiveMap.remove call').to.not.exist; + expect( + map.get('baz'), + 'Check non-removed keys are still present on a root after LiveMap.remove call for another keys', + ).to.equal(1); + }, + }, + + { + description: 'LiveMap.remove throws on invalid input', + action: async (ctx) => { + const { root, objectsHelper, channelName } = ctx; + + const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: objectsHelper.mapCreateRestOp(), + }); + await mapCreatedPromise; + + const map = root.get('map'); + + await expectToThrowAsync(async () => map.remove(), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(null), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(1), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(BigInt(1)), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(true), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(Symbol()), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove({}), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove([]), 'Map key should be string'); + await expectToThrowAsync(async () => map.remove(map), 'Map key should be string'); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'Objects.createCounter sends COUNTER_CREATE operation', + action: async (ctx) => { + const { objects } = ctx; + + const counters = await Promise.all(countersFixtures.map(async (x) => objects.createCounter(x.count))); + + for (let i = 0; i < counters.length; i++) { + const counter = counters[i]; + const fixture = countersFixtures[i]; + + expect(counter, `Check counter #${i + 1} exists`).to.exist; + expectInstanceOf(counter, 'LiveCounter', `Check counter instance #${i + 1} is of an expected class`); + expect(counter.value()).to.equal( + fixture.count ?? 0, + `Check counter #${i + 1} has expected initial value`, + ); + } + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveCounter created with Objects.createCounter can be assigned to the object tree', + action: async (ctx) => { + const { root, objects } = ctx; + + const counterCreatedPromise = waitForMapKeyUpdate(root, 'counter'); + const counter = await objects.createCounter(1); + await root.set('counter', counter); + await counterCreatedPromise; + + expectInstanceOf(counter, 'LiveCounter', `Check counter instance is of an expected class`); + expectInstanceOf( + root.get('counter'), + 'LiveCounter', + `Check counter instance on root is of an expected class`, + ); + expect(root.get('counter')).to.equal( + counter, + 'Check counter object on root is the same as from create method', + ); + expect(root.get('counter').value()).to.equal( + 1, + 'Check counter assigned to the object tree has the expected value', + ); + }, + }, + + { + description: + 'Objects.createCounter can return LiveCounter with initial value without applying CREATE operation', + action: async (ctx) => { + const { objects, helper } = ctx; + + // prevent publishing of ops to realtime so we guarantee that the initial value doesn't come from a CREATE op + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = () => {}; + + const counter = await objects.createCounter(1); + expect(counter.value()).to.equal(1, `Check counter has expected initial value`); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'Objects.createCounter can return LiveCounter with initial value from applied CREATE operation', + action: async (ctx) => { + const { objects, objectsHelper, helper, channel } = ctx; + + // instead of sending CREATE op to the realtime, echo it immediately to the client + // with forged initial value so we can check that counter gets initialized with a value from a CREATE op + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = async (objectMessages) => { + const counterId = objectMessages[0].operation.objectId; + // this should result execute regular operation application procedure and create an object in the pool with forged initial value + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [objectsHelper.counterCreateOp({ objectId: counterId, count: 10 })], + }); + }; + + const counter = await objects.createCounter(1); + + // counter should be created with forged initial value instead of the actual one + expect(counter.value()).to.equal( + 10, + 'Check counter value has the expected initial value from a CREATE operation', + ); + }, + }, + + { + description: + 'initial value is not double counted for LiveCounter from Objects.createCounter when CREATE op is received', + action: async (ctx) => { + const { objects, objectsHelper, helper, channel } = ctx; + + // prevent publishing of ops to realtime so we can guarantee order of operations + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = () => {}; + + // create counter locally, should have an initial value set + const counter = await objects.createCounter(1); + helper.recordPrivateApi('call.LiveObject.getObjectId'); + const counterId = counter.getObjectId(); + + // now inject CREATE op for a counter with a forged value. it should not be applied + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [objectsHelper.counterCreateOp({ objectId: counterId, count: 10 })], + }); + + expect(counter.value()).to.equal( + 1, + `Check counter initial value is not double counted after being created and receiving CREATE operation`, + ); + }, + }, + + { + description: 'Objects.createCounter throws on invalid input', + action: async (ctx) => { + const { root, objects } = ctx; + + await expectToThrowAsync(async () => objects.createCounter(null), 'Counter value should be a valid number'); + await expectToThrowAsync( + async () => objects.createCounter(Number.NaN), + 'Counter value should be a valid number', + ); + await expectToThrowAsync( + async () => objects.createCounter(Number.POSITIVE_INFINITY), + 'Counter value should be a valid number', + ); + await expectToThrowAsync( + async () => objects.createCounter(Number.NEGATIVE_INFINITY), + 'Counter value should be a valid number', + ); + await expectToThrowAsync( + async () => objects.createCounter('foo'), + 'Counter value should be a valid number', + ); + await expectToThrowAsync( + async () => objects.createCounter(BigInt(1)), + 'Counter value should be a valid number', + ); + await expectToThrowAsync(async () => objects.createCounter(true), 'Counter value should be a valid number'); + await expectToThrowAsync( + async () => objects.createCounter(Symbol()), + 'Counter value should be a valid number', + ); + await expectToThrowAsync(async () => objects.createCounter({}), 'Counter value should be a valid number'); + await expectToThrowAsync(async () => objects.createCounter([]), 'Counter value should be a valid number'); + await expectToThrowAsync(async () => objects.createCounter(root), 'Counter value should be a valid number'); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'Objects.createMap sends MAP_CREATE operation with primitive values', + action: async (ctx) => { + const { objects, helper } = ctx; + + const maps = await Promise.all( + primitiveMapsFixtures.map(async (mapFixture) => { + const entries = mapFixture.entries + ? Object.entries(mapFixture.entries).reduce((acc, [key, keyData]) => { + let value; + if (keyData.data.bytes != null) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + value = BufferUtils.base64Decode(keyData.data.bytes); + } else if (keyData.data.number != null) { + value = keyData.data.number; + } else if (keyData.data.string != null) { + value = keyData.data.string; + } else if (keyData.data.boolean != null) { + value = keyData.data.boolean; + } + + acc[key] = value; + return acc; + }, {}) + : undefined; + + return objects.createMap(entries); + }), + ); + + for (let i = 0; i < maps.length; i++) { + const map = maps[i]; + const fixture = primitiveMapsFixtures[i]; + + expect(map, `Check map #${i + 1} exists`).to.exist; + expectInstanceOf(map, 'LiveMap', `Check map instance #${i + 1} is of an expected class`); + + expect(map.size()).to.equal( + Object.keys(fixture.entries ?? {}).length, + `Check map #${i + 1} has correct number of keys`, + ); + + Object.entries(fixture.entries ?? {}).forEach(([key, keyData]) => { + if (keyData.data.bytes != null) { + helper.recordPrivateApi('call.BufferUtils.base64Decode'); + helper.recordPrivateApi('call.BufferUtils.areBuffersEqual'); + expect( + BufferUtils.areBuffersEqual(map.get(key), BufferUtils.base64Decode(keyData.data.bytes)), + `Check map #${i + 1} has correct value for "${key}" key`, + ).to.be.true; + } else { + const valueType = typeof map.get(key); + expect(map.get(key)).to.equal( + keyData.data[valueType], + `Check map #${i + 1} has correct value for "${key}" key`, + ); + } + }); + } + }, + }, + + { + allTransportsAndProtocols: true, + description: 'Objects.createMap sends MAP_CREATE operation with reference to another LiveObject', + action: async (ctx) => { + const { root, objectsHelper, channelName, objects } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'counter', + createOp: objectsHelper.counterCreateRestOp(), + }); + await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: 'map', + createOp: objectsHelper.mapCreateRestOp(), + }); + await objectsCreatedPromise; + + const counter = root.get('counter'); + const map = root.get('map'); + + const newMap = await objects.createMap({ counter, map }); + + expect(newMap, 'Check map exists').to.exist; + expectInstanceOf(newMap, 'LiveMap', 'Check map instance is of an expected class'); + + expect(newMap.get('counter')).to.equal( + counter, + 'Check can set a reference to a LiveCounter object on a new map via a MAP_CREATE operation', + ); + expect(newMap.get('map')).to.equal( + map, + 'Check can set a reference to a LiveMap object on a new map via a MAP_CREATE operation', + ); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'LiveMap created with Objects.createMap can be assigned to the object tree', + action: async (ctx) => { + const { root, objects } = ctx; + + const mapCreatedPromise = waitForMapKeyUpdate(root, 'map'); + const counter = await objects.createCounter(); + const map = await objects.createMap({ foo: 'bar', baz: counter }); + await root.set('map', map); + await mapCreatedPromise; + + expectInstanceOf(map, 'LiveMap', `Check map instance is of an expected class`); + expectInstanceOf(root.get('map'), 'LiveMap', `Check map instance on root is of an expected class`); + expect(root.get('map')).to.equal(map, 'Check map object on root is the same as from create method'); + expect(root.get('map').size()).to.equal( + 2, + 'Check map assigned to the object tree has the expected number of keys', + ); + expect(root.get('map').get('foo')).to.equal( + 'bar', + 'Check map assigned to the object tree has the expected value for its string key', + ); + expect(root.get('map').get('baz')).to.equal( + counter, + 'Check map assigned to the object tree has the expected value for its LiveCounter key', + ); + }, + }, + + { + description: 'Objects.createMap can return LiveMap with initial value without applying CREATE operation', + action: async (ctx) => { + const { objects, helper } = ctx; + + // prevent publishing of ops to realtime so we guarantee that the initial value doesn't come from a CREATE op + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = () => {}; + + const map = await objects.createMap({ foo: 'bar' }); + expect(map.get('foo')).to.equal('bar', `Check map has expected initial value`); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'Objects.createMap can return LiveMap with initial value from applied CREATE operation', + action: async (ctx) => { + const { objects, objectsHelper, helper, channel } = ctx; + + // instead of sending CREATE op to the realtime, echo it immediately to the client + // with forged initial value so we can check that map gets initialized with a value from a CREATE op + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = async (objectMessages) => { + const mapId = objectMessages[0].operation.objectId; + // this should result execute regular operation application procedure and create an object in the pool with forged initial value + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [ + objectsHelper.mapCreateOp({ + objectId: mapId, + entries: { baz: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { string: 'qux' } } }, + }), + ], + }); + }; + + const map = await objects.createMap({ foo: 'bar' }); + + // map should be created with forged initial value instead of the actual one + expect(map.get('foo'), `Check key "foo" was not set on a map client-side`).to.not.exist; + expect(map.get('baz')).to.equal( + 'qux', + `Check key "baz" was set on a map from a CREATE operation after object creation`, + ); + }, + }, + + { + description: + 'initial value is not double counted for LiveMap from Objects.createMap when CREATE op is received', + action: async (ctx) => { + const { objects, objectsHelper, helper, channel } = ctx; + + // prevent publishing of ops to realtime so we can guarantee order of operations + helper.recordPrivateApi('replace.Objects.publish'); + objects.publish = () => {}; + + // create map locally, should have an initial value set + const map = await objects.createMap({ foo: 'bar' }); + helper.recordPrivateApi('call.LiveObject.getObjectId'); + const mapId = map.getObjectId(); + + // now inject CREATE op for a map with a forged value. it should not be applied + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 1, 1), + siteCode: 'aaa', + state: [ + objectsHelper.mapCreateOp({ + objectId: mapId, + entries: { + foo: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { string: 'qux' } }, + baz: { timeserial: lexicoTimeserial('aaa', 1, 1), data: { string: 'qux' } }, + }, + }), + ], + }); + + expect(map.get('foo')).to.equal( + 'bar', + `Check key "foo" was not overridden by a CREATE operation after creating a map locally`, + ); + expect(map.get('baz'), `Check key "baz" was not set by a CREATE operation after creating a map locally`).to + .not.exist; + }, + }, + + { + description: 'Objects.createMap throws on invalid input', + action: async (ctx) => { + const { root, objects } = ctx; + + await expectToThrowAsync(async () => objects.createMap(null), 'Map entries should be a key-value object'); + await expectToThrowAsync(async () => objects.createMap('foo'), 'Map entries should be a key-value object'); + await expectToThrowAsync(async () => objects.createMap(1), 'Map entries should be a key-value object'); + await expectToThrowAsync( + async () => objects.createMap(BigInt(1)), + 'Map entries should be a key-value object', + ); + await expectToThrowAsync(async () => objects.createMap(true), 'Map entries should be a key-value object'); + await expectToThrowAsync( + async () => objects.createMap(Symbol()), + 'Map entries should be a key-value object', + ); + + await expectToThrowAsync( + async () => objects.createMap({ key: undefined }), + 'Map value data type is unsupported', + ); + await expectToThrowAsync( + async () => objects.createMap({ key: null }), + 'Map value data type is unsupported', + ); + await expectToThrowAsync( + async () => objects.createMap({ key: BigInt(1) }), + 'Map value data type is unsupported', + ); + await expectToThrowAsync( + async () => objects.createMap({ key: Symbol() }), + 'Map value data type is unsupported', + ); + await expectToThrowAsync(async () => objects.createMap({ key: {} }), 'Map value data type is unsupported'); + await expectToThrowAsync(async () => objects.createMap({ key: [] }), 'Map value data type is unsupported'); + }, + }, + + { + description: 'batch API getRoot method is synchronous', + action: async (ctx) => { + const { objects } = ctx; + + await objects.batch((ctx) => { + const root = ctx.getRoot(); + expect(root, 'Check getRoot method in a BatchContext returns root object synchronously').to.exist; + expectInstanceOf(root, 'LiveMap', 'root object obtained from a BatchContext is a LiveMap'); + }); + }, + }, + + { + description: 'batch API .get method on a map returns BatchContext* wrappers for objects', + action: async (ctx) => { + const { root, objects } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ innerCounter: counter }); + await root.set('counter', counter); + await root.set('map', map); + await objectsCreatedPromise; + + await objects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + const ctxCounter = ctxRoot.get('counter'); + const ctxMap = ctxRoot.get('map'); + const ctxInnerCounter = ctxMap.get('innerCounter'); + + expect(ctxCounter, 'Check counter object can be accessed from a map in a batch API').to.exist; + expectInstanceOf( + ctxCounter, + 'BatchContextLiveCounter', + 'Check counter object obtained in a batch API has a BatchContext specific wrapper type', + ); + expect(ctxMap, 'Check map object can be accessed from a map in a batch API').to.exist; + expectInstanceOf( + ctxMap, + 'BatchContextLiveMap', + 'Check map object obtained in a batch API has a BatchContext specific wrapper type', + ); + expect(ctxInnerCounter, 'Check inner counter object can be accessed from a map in a batch API').to.exist; + expectInstanceOf( + ctxInnerCounter, + 'BatchContextLiveCounter', + 'Check inner counter object obtained in a batch API has a BatchContext specific wrapper type', + ); + }); + }, + }, + + { + description: 'batch API access API methods on objects work and are synchronous', + action: async (ctx) => { + const { root, objects } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + await objectsCreatedPromise; + + await objects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + const ctxCounter = ctxRoot.get('counter'); + const ctxMap = ctxRoot.get('map'); + + expect(ctxCounter.value()).to.equal( + 1, + 'Check batch API counter .value() method works and is synchronous', + ); + expect(ctxMap.get('foo')).to.equal('bar', 'Check batch API map .get() method works and is synchronous'); + expect(ctxMap.size()).to.equal(1, 'Check batch API map .size() method works and is synchronous'); + expect([...ctxMap.entries()]).to.deep.equal( + [['foo', 'bar']], + 'Check batch API map .entries() method works and is synchronous', + ); + expect([...ctxMap.keys()]).to.deep.equal( + ['foo'], + 'Check batch API map .keys() method works and is synchronous', + ); + expect([...ctxMap.values()]).to.deep.equal( + ['bar'], + 'Check batch API map .values() method works and is synchronous', + ); + }); + }, + }, + + { + description: 'batch API write API methods on objects do not mutate objects inside the batch callback', + action: async (ctx) => { + const { root, objects } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + await objectsCreatedPromise; + + await objects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + const ctxCounter = ctxRoot.get('counter'); + const ctxMap = ctxRoot.get('map'); + + ctxCounter.increment(10); + expect(ctxCounter.value()).to.equal( + 1, + 'Check batch API counter .increment method does not mutate the object inside the batch callback', + ); + + ctxCounter.decrement(100); + expect(ctxCounter.value()).to.equal( + 1, + 'Check batch API counter .decrement method does not mutate the object inside the batch callback', + ); + + ctxMap.set('baz', 'qux'); + expect( + ctxMap.get('baz'), + 'Check batch API map .set method does not mutate the object inside the batch callback', + ).to.not.exist; + + ctxMap.remove('foo'); + expect(ctxMap.get('foo')).to.equal( + 'bar', + 'Check batch API map .remove method does not mutate the object inside the batch callback', + ); + }); + }, + }, + + { + allTransportsAndProtocols: true, + description: 'batch API scheduled operations are applied when batch callback is finished', + action: async (ctx) => { + const { root, objects } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + await objectsCreatedPromise; + + await objects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + const ctxCounter = ctxRoot.get('counter'); + const ctxMap = ctxRoot.get('map'); + + ctxCounter.increment(10); + ctxCounter.decrement(100); + + ctxMap.set('baz', 'qux'); + ctxMap.remove('foo'); + }); + + expect(counter.value()).to.equal(1 + 10 - 100, 'Check counter has an expected value after batch call'); + expect(map.get('baz')).to.equal('qux', 'Check key "baz" has an expected value in a map after batch call'); + expect(map.get('foo'), 'Check key "foo" is removed from map after batch call').to.not.exist; + }, + }, + + { + description: 'batch API can be called without scheduling any operations', + action: async (ctx) => { + const { objects } = ctx; + + let caughtError; + try { + await objects.batch((ctx) => {}); + } catch (error) { + caughtError = error; + } + expect( + caughtError, + `Check batch API can be called without scheduling any operations, but got error: ${caughtError?.toString()}`, + ).to.not.exist; + }, + }, + + { + description: 'batch API scheduled operations can be canceled by throwing an error in the batch callback', + action: async (ctx) => { + const { root, objects } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + await objectsCreatedPromise; + + const cancelError = new Error('cancel batch'); + let caughtError; + try { + await objects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + const ctxCounter = ctxRoot.get('counter'); + const ctxMap = ctxRoot.get('map'); + + ctxCounter.increment(10); + ctxCounter.decrement(100); + + ctxMap.set('baz', 'qux'); + ctxMap.remove('foo'); + + throw cancelError; + }); + } catch (error) { + caughtError = error; + } + + expect(counter.value()).to.equal(1, 'Check counter value is not changed after canceled batch call'); + expect(map.get('baz'), 'Check key "baz" does not exist on a map after canceled batch call').to.not.exist; + expect(map.get('foo')).to.equal('bar', 'Check key "foo" is not changed on a map after canceled batch call'); + expect(caughtError).to.equal( + cancelError, + 'Check error from a batch callback was rethrown by a batch method', + ); + }, + }, + + { + description: `batch API batch context and derived objects can't be interacted with after the batch call`, + action: async (ctx) => { + const { root, objects } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + await objectsCreatedPromise; + + let savedCtx; + let savedCtxCounter; + let savedCtxMap; + + await objects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + savedCtx = ctx; + savedCtxCounter = ctxRoot.get('counter'); + savedCtxMap = ctxRoot.get('map'); + }); + + expectAccessBatchApiToThrow({ + ctx: savedCtx, + map: savedCtxMap, + counter: savedCtxCounter, + errorMsg: 'Batch is closed', + }); + expectWriteBatchApiToThrow({ + ctx: savedCtx, + map: savedCtxMap, + counter: savedCtxCounter, + errorMsg: 'Batch is closed', + }); + }, + }, + + { + description: `batch API batch context and derived objects can't be interacted with after error was thrown from batch callback`, + action: async (ctx) => { + const { root, objects } = ctx; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'counter'), + waitForMapKeyUpdate(root, 'map'), + ]); + const counter = await objects.createCounter(1); + const map = await objects.createMap({ foo: 'bar' }); + await root.set('counter', counter); + await root.set('map', map); + await objectsCreatedPromise; + + let savedCtx; + let savedCtxCounter; + let savedCtxMap; + + let caughtError; + try { + await objects.batch((ctx) => { + const ctxRoot = ctx.getRoot(); + savedCtx = ctx; + savedCtxCounter = ctxRoot.get('counter'); + savedCtxMap = ctxRoot.get('map'); + + throw new Error('cancel batch'); + }); + } catch (error) { + caughtError = error; + } + + expect(caughtError, 'Check batch call failed with an error').to.exist; + expectAccessBatchApiToThrow({ + ctx: savedCtx, + map: savedCtxMap, + counter: savedCtxCounter, + errorMsg: 'Batch is closed', + }); + expectWriteBatchApiToThrow({ + ctx: savedCtx, + map: savedCtxMap, + counter: savedCtxCounter, + errorMsg: 'Batch is closed', + }); + }, + }, + ]; + + const liveMapEnumerationScenarios = [ + { + description: `LiveMap enumeration`, + action: async (ctx) => { + const { root, objectsHelper, channel } = ctx; + + const counterId1 = objectsHelper.fakeCounterObjectId(); + const counterId2 = objectsHelper.fakeCounterObjectId(); + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:', // empty serial so sync sequence ends immediately + state: [ + objectsHelper.counterObject({ + objectId: counterId1, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: false, + initialCount: 0, + }), + objectsHelper.counterObject({ + objectId: counterId2, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialCount: 0, + }), + objectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + materialisedEntries: { + counter1: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId1 } }, + counter2: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId2 } }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'bar' } }, + baz: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'qux' }, tombstone: true }, + }, + }), + ], + }); + + const counter1 = await root.get('counter1'); + + // enumeration methods should not count tombstoned entries + expect(root.size()).to.equal(2, 'Check LiveMap.size() returns expected number of keys'); + expect([...root.entries()]).to.deep.equal( + [ + ['counter1', counter1], + ['foo', 'bar'], + ], + 'Check LiveMap.entries() returns expected entries', + ); + expect([...root.keys()]).to.deep.equal(['counter1', 'foo'], 'Check LiveMap.keys() returns expected keys'); + expect([...root.values()]).to.deep.equal( + [counter1, 'bar'], + 'Check LiveMap.values() returns expected values', + ); + }, + }, + { + description: `BatchContextLiveMap enumeration`, + action: async (ctx) => { + const { root, objectsHelper, channel, objects } = ctx; + + const counterId1 = objectsHelper.fakeCounterObjectId(); + const counterId2 = objectsHelper.fakeCounterObjectId(); + await objectsHelper.processObjectStateMessageOnChannel({ + channel, + syncSerial: 'serial:', // empty serial so sync sequence ends immediately + state: [ + objectsHelper.counterObject({ + objectId: counterId1, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: false, + initialCount: 0, + }), + objectsHelper.counterObject({ + objectId: counterId2, + siteTimeserials: { + aaa: lexicoTimeserial('aaa', 0, 0), + }, + tombstone: true, + initialCount: 0, + }), + objectsHelper.mapObject({ + objectId: 'root', + siteTimeserials: { aaa: lexicoTimeserial('aaa', 0, 0) }, + materialisedEntries: { + counter1: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId1 } }, + counter2: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { objectId: counterId2 } }, + foo: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'bar' } }, + baz: { timeserial: lexicoTimeserial('aaa', 0, 0), data: { string: 'qux' }, tombstone: true }, + }, + }), + ], + }); + + const counter1 = await root.get('counter1'); + + await objects.batch(async (ctx) => { + const ctxRoot = ctx.getRoot(); + + // enumeration methods should not count tombstoned entries + expect(ctxRoot.size()).to.equal(2, 'Check BatchContextLiveMap.size() returns expected number of keys'); + expect([...ctxRoot.entries()]).to.deep.equal( + [ + ['counter1', counter1], + ['foo', 'bar'], + ], + 'Check BatchContextLiveMap.entries() returns expected entries', + ); + expect([...ctxRoot.keys()]).to.deep.equal( + ['counter1', 'foo'], + 'Check BatchContextLiveMap.keys() returns expected keys', + ); + expect([...ctxRoot.values()]).to.deep.equal( + [counter1, 'bar'], + 'Check BatchContextLiveMap.values() returns expected values', + ); + }); + }, + }, + ]; + + /** @nospec */ + forScenarios( + this, + [ + ...objectSyncSequenceScenarios, + ...applyOperationsScenarios, + ...applyOperationsDuringSyncScenarios, + ...writeApiScenarios, + ...liveMapEnumerationScenarios, + ], + async function (helper, scenario, clientOptions, channelName) { + const objectsHelper = new ObjectsHelper(helper); + const client = RealtimeWithObjects(helper, clientOptions); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const channel = client.channels.get(channelName, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + await scenario.action({ + objects, + root, + objectsHelper, + channelName, + channel, + client, + helper, + clientOptions, + }); + }, client); + }, + ); + + const subscriptionCallbacksScenarios = [ + { + allTransportsAndProtocols: true, + description: 'can subscribe to the incoming COUNTER_INC operation on a LiveCounter', + action: async (ctx) => { + const { root, objectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + + const counter = root.get(sampleCounterKey); + const subscriptionPromise = new Promise((resolve, reject) => + counter.subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { amount: 1 } }, + 'Check counter subscription callback is called with an expected update object for COUNTER_INC operation', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + + await objectsHelper.operationRequest( + channelName, + objectsHelper.counterIncRestOp({ + objectId: sampleCounterObjectId, + number: 1, + }), + ); + + await subscriptionPromise; + }, + }, + + { + allTransportsAndProtocols: true, + description: 'can subscribe to multiple incoming operations on a LiveCounter', + action: async (ctx) => { + const { root, objectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + + const counter = root.get(sampleCounterKey); + const expectedCounterIncrements = [100, -100, Number.MAX_SAFE_INTEGER, -Number.MAX_SAFE_INTEGER]; + let currentUpdateIndex = 0; + + const subscriptionPromise = new Promise((resolve, reject) => + counter.subscribe((update) => { + try { + const expectedInc = expectedCounterIncrements[currentUpdateIndex]; + expect(update).to.deep.equal( + { update: { amount: expectedInc } }, + `Check counter subscription callback is called with an expected update object for ${currentUpdateIndex + 1} times`, + ); + + if (currentUpdateIndex === expectedCounterIncrements.length - 1) { + resolve(); + } + + currentUpdateIndex++; + } catch (error) { + reject(error); + } + }), + ); + + for (const increment of expectedCounterIncrements) { + await objectsHelper.operationRequest( + channelName, + objectsHelper.counterIncRestOp({ + objectId: sampleCounterObjectId, + number: increment, + }), + ); + } + + await subscriptionPromise; + }, + }, + + { + allTransportsAndProtocols: true, + description: 'can subscribe to the incoming MAP_SET operation on a LiveMap', + action: async (ctx) => { + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + const subscriptionPromise = new Promise((resolve, reject) => + map.subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { stringKey: 'updated' } }, + 'Check map subscription callback is called with an expected update object for MAP_SET operation', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapSetRestOp({ + objectId: sampleMapObjectId, + key: 'stringKey', + value: { string: 'stringValue' }, + }), + ); + + await subscriptionPromise; + }, + }, + + { + allTransportsAndProtocols: true, + description: 'can subscribe to the incoming MAP_REMOVE operation on a LiveMap', + action: async (ctx) => { + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + const subscriptionPromise = new Promise((resolve, reject) => + map.subscribe((update) => { + try { + expect(update).to.deep.equal( + { update: { stringKey: 'removed' } }, + 'Check map subscription callback is called with an expected update object for MAP_REMOVE operation', + ); + resolve(); + } catch (error) { + reject(error); + } + }), + ); + + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapRemoveRestOp({ + objectId: sampleMapObjectId, + key: 'stringKey', + }), + ); + + await subscriptionPromise; + }, + }, + + { + allTransportsAndProtocols: true, + description: 'can subscribe to multiple incoming operations on a LiveMap', + action: async (ctx) => { + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + const expectedMapUpdates = [ + { update: { foo: 'updated' } }, + { update: { bar: 'updated' } }, + { update: { foo: 'removed' } }, + { update: { baz: 'updated' } }, + { update: { bar: 'removed' } }, + ]; + let currentUpdateIndex = 0; + + const subscriptionPromise = new Promise((resolve, reject) => + map.subscribe((update) => { + try { + expect(update).to.deep.equal( + expectedMapUpdates[currentUpdateIndex], + `Check map subscription callback is called with an expected update object for ${currentUpdateIndex + 1} times`, + ); + + if (currentUpdateIndex === expectedMapUpdates.length - 1) { + resolve(); + } + + currentUpdateIndex++; + } catch (error) { + reject(error); + } + }), + ); + + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapSetRestOp({ + objectId: sampleMapObjectId, + key: 'foo', + value: { string: 'something' }, + }), + ); + + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapSetRestOp({ + objectId: sampleMapObjectId, + key: 'bar', + value: { string: 'something' }, + }), + ); + + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapRemoveRestOp({ + objectId: sampleMapObjectId, + key: 'foo', + }), + ); + + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapSetRestOp({ + objectId: sampleMapObjectId, + key: 'baz', + value: { string: 'something' }, + }), + ); + + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapRemoveRestOp({ + objectId: sampleMapObjectId, + key: 'bar', + }), + ); + + await subscriptionPromise; + }, + }, + + { + description: 'can unsubscribe from LiveCounter updates via returned "unsubscribe" callback', + action: async (ctx) => { + const { root, objectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + + const counter = root.get(sampleCounterKey); + let callbackCalled = 0; + const subscriptionPromise = new Promise((resolve) => { + const { unsubscribe } = counter.subscribe(() => { + callbackCalled++; + // unsubscribe from future updates after the first call + unsubscribe(); + resolve(); + }); + }); + + const increments = 3; + for (let i = 0; i < increments; i++) { + const counterUpdatedPromise = waitForCounterUpdate(counter); + await objectsHelper.operationRequest( + channelName, + objectsHelper.counterIncRestOp({ + objectId: sampleCounterObjectId, + number: 1, + }), + ); + await counterUpdatedPromise; + } + + await subscriptionPromise; + + expect(counter.value()).to.equal(3, 'Check counter has final expected value after all increments'); + expect(callbackCalled).to.equal(1, 'Check subscription callback was only called once'); + }, + }, + + { + description: 'can unsubscribe from LiveCounter updates via LiveCounter.unsubscribe() call', + action: async (ctx) => { + const { root, objectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + + const counter = root.get(sampleCounterKey); + let callbackCalled = 0; + const subscriptionPromise = new Promise((resolve) => { + const listener = () => { + callbackCalled++; + // unsubscribe from future updates after the first call + counter.unsubscribe(listener); + resolve(); + }; + + counter.subscribe(listener); + }); + + const increments = 3; + for (let i = 0; i < increments; i++) { + const counterUpdatedPromise = waitForCounterUpdate(counter); + await objectsHelper.operationRequest( + channelName, + objectsHelper.counterIncRestOp({ + objectId: sampleCounterObjectId, + number: 1, + }), + ); + await counterUpdatedPromise; + } + + await subscriptionPromise; + + expect(counter.value()).to.equal(3, 'Check counter has final expected value after all increments'); + expect(callbackCalled).to.equal(1, 'Check subscription callback was only called once'); + }, + }, + + { + description: 'can remove all LiveCounter update listeners via LiveCounter.unsubscribeAll() call', + action: async (ctx) => { + const { root, objectsHelper, channelName, sampleCounterKey, sampleCounterObjectId } = ctx; + + const counter = root.get(sampleCounterKey); + const callbacks = 3; + const callbacksCalled = new Array(callbacks).fill(0); + const subscriptionPromises = []; + + for (let i = 0; i < callbacks; i++) { + const promise = new Promise((resolve) => { + counter.subscribe(() => { + callbacksCalled[i]++; + resolve(); + }); + }); + subscriptionPromises.push(promise); + } + + const increments = 3; + for (let i = 0; i < increments; i++) { + const counterUpdatedPromise = waitForCounterUpdate(counter); + await objectsHelper.operationRequest( + channelName, + objectsHelper.counterIncRestOp({ + objectId: sampleCounterObjectId, + number: 1, + }), + ); + await counterUpdatedPromise; + + if (i === 0) { + // unsub all after first operation + counter.unsubscribeAll(); + } + } + + await Promise.all(subscriptionPromises); + + expect(counter.value()).to.equal(3, 'Check counter has final expected value after all increments'); + callbacksCalled.forEach((x) => expect(x).to.equal(1, 'Check subscription callbacks were called once each')); + }, + }, + + { + description: 'can unsubscribe from LiveMap updates via returned "unsubscribe" callback', + action: async (ctx) => { + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + let callbackCalled = 0; + const subscriptionPromise = new Promise((resolve) => { + const { unsubscribe } = map.subscribe(() => { + callbackCalled++; + // unsubscribe from future updates after the first call + unsubscribe(); + resolve(); + }); + }); + + const mapSets = 3; + for (let i = 0; i < mapSets; i++) { + const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapSetRestOp({ + objectId: sampleMapObjectId, + key: `foo-${i}`, + value: { string: 'exists' }, + }), + ); + await mapUpdatedPromise; + } + + await subscriptionPromise; + + for (let i = 0; i < mapSets; i++) { + expect(map.get(`foo-${i}`)).to.equal( + 'exists', + `Check map has value for key "foo-${i}" after all map sets`, + ); + } + expect(callbackCalled).to.equal(1, 'Check subscription callback was only called once'); + }, + }, + + { + description: 'can unsubscribe from LiveMap updates via LiveMap.unsubscribe() call', + action: async (ctx) => { + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + let callbackCalled = 0; + const subscriptionPromise = new Promise((resolve) => { + const listener = () => { + callbackCalled++; + // unsubscribe from future updates after the first call + map.unsubscribe(listener); + resolve(); + }; + + map.subscribe(listener); + }); + + const mapSets = 3; + for (let i = 0; i < mapSets; i++) { + const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapSetRestOp({ + objectId: sampleMapObjectId, + key: `foo-${i}`, + value: { string: 'exists' }, + }), + ); + await mapUpdatedPromise; + } + + await subscriptionPromise; + + for (let i = 0; i < mapSets; i++) { + expect(map.get(`foo-${i}`)).to.equal( + 'exists', + `Check map has value for key "foo-${i}" after all map sets`, + ); + } + expect(callbackCalled).to.equal(1, 'Check subscription callback was only called once'); + }, + }, + + { + description: 'can remove all LiveMap update listeners via LiveMap.unsubscribeAll() call', + action: async (ctx) => { + const { root, objectsHelper, channelName, sampleMapKey, sampleMapObjectId } = ctx; + + const map = root.get(sampleMapKey); + const callbacks = 3; + const callbacksCalled = new Array(callbacks).fill(0); + const subscriptionPromises = []; + + for (let i = 0; i < callbacks; i++) { + const promise = new Promise((resolve) => { + map.subscribe(() => { + callbacksCalled[i]++; + resolve(); + }); + }); + subscriptionPromises.push(promise); + } + + const mapSets = 3; + for (let i = 0; i < mapSets; i++) { + const mapUpdatedPromise = waitForMapKeyUpdate(map, `foo-${i}`); + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapSetRestOp({ + objectId: sampleMapObjectId, + key: `foo-${i}`, + value: { string: 'exists' }, + }), + ); + await mapUpdatedPromise; + + if (i === 0) { + // unsub all after first operation + map.unsubscribeAll(); + } + } + + await Promise.all(subscriptionPromises); + + for (let i = 0; i < mapSets; i++) { + expect(map.get(`foo-${i}`)).to.equal( + 'exists', + `Check map has value for key "foo-${i}" after all map sets`, + ); + } + callbacksCalled.forEach((x) => expect(x).to.equal(1, 'Check subscription callbacks were called once each')); + }, + }, + ]; + + /** @nospec */ + forScenarios(this, subscriptionCallbacksScenarios, async function (helper, scenario, clientOptions, channelName) { + const objectsHelper = new ObjectsHelper(helper); + const client = RealtimeWithObjects(helper, clientOptions); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const channel = client.channels.get(channelName, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + const sampleMapKey = 'sampleMap'; + const sampleCounterKey = 'sampleCounter'; + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, sampleMapKey), + waitForMapKeyUpdate(root, sampleCounterKey), + ]); + // prepare map and counter objects for use by the scenario + const { objectId: sampleMapObjectId } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: sampleMapKey, + createOp: objectsHelper.mapCreateRestOp(), + }); + const { objectId: sampleCounterObjectId } = await objectsHelper.createAndSetOnMap(channelName, { + mapObjectId: 'root', + key: sampleCounterKey, + createOp: objectsHelper.counterCreateRestOp(), + }); + await objectsCreatedPromise; + + await scenario.action({ + root, + objectsHelper, + channelName, + channel, + sampleMapKey, + sampleMapObjectId, + sampleCounterKey, + sampleCounterObjectId, + }); + }, client); + }); + + const tombstonesGCScenarios = [ + // for the next tests we need to access the private API of Objects plugin in order to verify that tombstoned entities were indeed deleted after the GC grace period. + // public API hides that kind of information from the user and returns undefined for tombstoned entities even if realtime client still keeps a reference to them. + { + description: 'tombstoned object is removed from the pool after the GC grace period', + action: async (ctx) => { + const { objectsHelper, channelName, channel, objects, helper, waitForGCCycles, client } = ctx; + + const counterCreatedPromise = waitForObjectOperation(helper, client, ObjectsHelper.ACTIONS.COUNTER_CREATE); + // send a CREATE op, this adds an object to the pool + const { objectId } = await objectsHelper.operationRequest( + channelName, + objectsHelper.counterCreateRestOp({ number: 1 }), + ); + await counterCreatedPromise; + + helper.recordPrivateApi('call.Objects._objectsPool.get'); + expect(objects._objectsPool.get(objectId), 'Check object exists in the pool after creation').to.exist; + + // inject OBJECT_DELETE for the object. this should tombstone the object and make it inaccessible to the end user, but still keep it in memory in the local pool + await objectsHelper.processObjectOperationMessageOnChannel({ + channel, + serial: lexicoTimeserial('aaa', 0, 0), + siteCode: 'aaa', + state: [objectsHelper.objectDeleteOp({ objectId })], + }); + + helper.recordPrivateApi('call.Objects._objectsPool.get'); + expect( + objects._objectsPool.get(objectId), + 'Check object exists in the pool immediately after OBJECT_DELETE', + ).to.exist; + helper.recordPrivateApi('call.Objects._objectsPool.get'); + helper.recordPrivateApi('call.LiveObject.isTombstoned'); + expect(objects._objectsPool.get(objectId).isTombstoned()).to.equal( + true, + `Check object's "tombstone" flag is set to "true" after OBJECT_DELETE`, + ); + + // we expect 2 cycles to guarantee that grace period has expired, which will always be true based on the test config used + await waitForGCCycles(2); + + // object should be removed from the local pool entirely now, as the GC grace period has passed + helper.recordPrivateApi('call.Objects._objectsPool.get'); + expect( + objects._objectsPool.get(objectId), + 'Check object exists does not exist in the pool after the GC grace period expiration', + ).to.not.exist; + }, + }, + + { + allTransportsAndProtocols: true, + description: 'tombstoned map entry is removed from the LiveMap after the GC grace period', + action: async (ctx) => { + const { root, objectsHelper, channelName, helper, waitForGCCycles } = ctx; + + const keyUpdatedPromise = waitForMapKeyUpdate(root, 'foo'); + // set a key on a root + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapSetRestOp({ objectId: 'root', key: 'foo', value: { string: 'bar' } }), + ); + await keyUpdatedPromise; + + expect(root.get('foo')).to.equal('bar', 'Check key "foo" exists on root after MAP_SET'); + + const keyUpdatedPromise2 = waitForMapKeyUpdate(root, 'foo'); + // remove the key from the root. this should tombstone the map entry and make it inaccessible to the end user, but still keep it in memory in the underlying map + await objectsHelper.operationRequest( + channelName, + objectsHelper.mapRemoveRestOp({ objectId: 'root', key: 'foo' }), + ); + await keyUpdatedPromise2; + + expect(root.get('foo'), 'Check key "foo" is inaccessible via public API on root after MAP_REMOVE').to.not + .exist; + helper.recordPrivateApi('read.LiveMap._dataRef.data'); + expect( + root._dataRef.data.get('foo'), + 'Check map entry for "foo" exists on root in the underlying data immediately after MAP_REMOVE', + ).to.exist; + helper.recordPrivateApi('read.LiveMap._dataRef.data'); + expect( + root._dataRef.data.get('foo').tombstone, + 'Check map entry for "foo" on root has "tombstone" flag set to "true" after MAP_REMOVE', + ).to.exist; + + // we expect 2 cycles to guarantee that grace period has expired, which will always be true based on the test config used + await waitForGCCycles(2); + + // the entry should be removed from the underlying map now + helper.recordPrivateApi('read.LiveMap._dataRef.data'); + expect( + root._dataRef.data.get('foo'), + 'Check map entry for "foo" does not exist on root in the underlying data after the GC grace period expiration', + ).to.not.exist; + }, + }, + ]; + + /** @nospec */ + forScenarios(this, tombstonesGCScenarios, async function (helper, scenario, clientOptions, channelName) { + try { + helper.recordPrivateApi('write.Objects._DEFAULTS.gcInterval'); + ObjectsPlugin.Objects._DEFAULTS.gcInterval = 500; + helper.recordPrivateApi('write.Objects._DEFAULTS.gcGracePeriod'); + ObjectsPlugin.Objects._DEFAULTS.gcGracePeriod = 250; + + const objectsHelper = new ObjectsHelper(helper); + const client = RealtimeWithObjects(helper, clientOptions); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const channel = client.channels.get(channelName, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + // helper function to spy on the GC interval callback and wait for a specific number of GC cycles. + // returns a promise which will resolve when required number of cycles have happened. + const waitForGCCycles = (cycles) => { + const onGCIntervalOriginal = objects._objectsPool._onGCInterval; + let gcCalledTimes = 0; + return new Promise((resolve) => { + helper.recordPrivateApi('replace.Objects._objectsPool._onGCInterval'); + objects._objectsPool._onGCInterval = function () { + helper.recordPrivateApi('call.Objects._objectsPool._onGCInterval'); + onGCIntervalOriginal.call(this); + + gcCalledTimes++; + if (gcCalledTimes >= cycles) { + resolve(); + objects._objectsPool._onGCInterval = onGCIntervalOriginal; + } + }; + }); + }; + + await scenario.action({ + client, + root, + objectsHelper, + channelName, + channel, + objects, + helper, + waitForGCCycles, + }); + }, client); + } finally { + helper.recordPrivateApi('write.Objects._DEFAULTS.gcInterval'); + ObjectsPlugin.Objects._DEFAULTS.gcInterval = gcIntervalOriginal; + helper.recordPrivateApi('write.Objects._DEFAULTS.gcGracePeriod'); + ObjectsPlugin.Objects._DEFAULTS.gcGracePeriod = gcGracePeriodOriginal; + } + }); + + const expectAccessApiToThrow = async ({ objects, map, counter, errorMsg }) => { + await expectToThrowAsync(async () => objects.getRoot(), errorMsg); + + expect(() => counter.value()).to.throw(errorMsg); + + expect(() => map.get()).to.throw(errorMsg); + expect(() => map.size()).to.throw(errorMsg); + expect(() => [...map.entries()]).to.throw(errorMsg); + expect(() => [...map.keys()]).to.throw(errorMsg); + expect(() => [...map.values()]).to.throw(errorMsg); + + for (const obj of [map, counter]) { + expect(() => obj.subscribe()).to.throw(errorMsg); + expect(() => obj.unsubscribe(() => {})).not.to.throw(); // this should not throw + expect(() => obj.unsubscribeAll()).not.to.throw(); // this should not throw + } + }; + + const expectWriteApiToThrow = async ({ objects, map, counter, errorMsg }) => { + await expectToThrowAsync(async () => objects.batch(), errorMsg); + await expectToThrowAsync(async () => objects.createMap(), errorMsg); + await expectToThrowAsync(async () => objects.createCounter(), errorMsg); + + await expectToThrowAsync(async () => counter.increment(), errorMsg); + await expectToThrowAsync(async () => counter.decrement(), errorMsg); + + await expectToThrowAsync(async () => map.set(), errorMsg); + await expectToThrowAsync(async () => map.remove(), errorMsg); + + for (const obj of [map, counter]) { + expect(() => obj.unsubscribe(() => {})).not.to.throw(); // this should not throw + expect(() => obj.unsubscribeAll()).not.to.throw(); // this should not throw + } + }; + + /** Make sure to call this inside the batch method as batch objects can't be interacted with outside the batch callback */ + const expectAccessBatchApiToThrow = ({ ctx, map, counter, errorMsg }) => { + expect(() => ctx.getRoot()).to.throw(errorMsg); + + expect(() => counter.value()).to.throw(errorMsg); + + expect(() => map.get()).to.throw(errorMsg); + expect(() => map.size()).to.throw(errorMsg); + expect(() => [...map.entries()]).to.throw(errorMsg); + expect(() => [...map.keys()]).to.throw(errorMsg); + expect(() => [...map.values()]).to.throw(errorMsg); + }; + + /** Make sure to call this inside the batch method as batch objects can't be interacted with outside the batch callback */ + const expectWriteBatchApiToThrow = ({ ctx, map, counter, errorMsg }) => { + expect(() => counter.increment()).to.throw(errorMsg); + expect(() => counter.decrement()).to.throw(errorMsg); + + expect(() => map.set()).to.throw(errorMsg); + expect(() => map.remove()).to.throw(errorMsg); + }; + + const clientConfigurationScenarios = [ + { + description: 'public API throws missing object modes error when attached without correct modes', + action: async (ctx) => { + const { objects, channel, map, counter } = ctx; + + // obtain batch context with valid modes first + await objects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate missing modes + channel.modes = []; + + expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: '"object_subscribe" channel mode' }); + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: '"object_publish" channel mode' }); + }); + + await expectAccessApiToThrow({ objects, map, counter, errorMsg: '"object_subscribe" channel mode' }); + await expectWriteApiToThrow({ objects, map, counter, errorMsg: '"object_publish" channel mode' }); + }, + }, + + { + description: + 'public API throws missing object modes error when not yet attached but client options are missing correct modes', + action: async (ctx) => { + const { objects, channel, map, counter, helper } = ctx; + + // obtain batch context with valid modes first + await objects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate a situation where we're not yet attached/modes are not received on ATTACHED event + channel.modes = undefined; + helper.recordPrivateApi('write.channel.channelOptions.modes'); + channel.channelOptions.modes = []; + + expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: '"object_subscribe" channel mode' }); + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: '"object_publish" channel mode' }); + }); + + await expectAccessApiToThrow({ objects, map, counter, errorMsg: '"object_subscribe" channel mode' }); + await expectWriteApiToThrow({ objects, map, counter, errorMsg: '"object_publish" channel mode' }); + }, + }, + + { + description: 'public API throws invalid channel state error when channel DETACHED', + action: async (ctx) => { + const { objects, channel, map, counter, helper } = ctx; + + // obtain batch context with valid channel state first + await objects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate channel state change + helper.recordPrivateApi('call.channel.requestState'); + channel.requestState('detached'); + + expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: 'failed as channel state is detached' }); + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: 'failed as channel state is detached' }); + }); + + await expectAccessApiToThrow({ + objects, + map, + counter, + errorMsg: 'failed as channel state is detached', + }); + await expectWriteApiToThrow({ objects, map, counter, errorMsg: 'failed as channel state is detached' }); + }, + }, + + { + description: 'public API throws invalid channel state error when channel FAILED', + action: async (ctx) => { + const { objects, channel, map, counter, helper } = ctx; + + // obtain batch context with valid channel state first + await objects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate channel state change + helper.recordPrivateApi('call.channel.requestState'); + channel.requestState('failed'); + + expectAccessBatchApiToThrow({ ctx, map, counter, errorMsg: 'failed as channel state is failed' }); + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: 'failed as channel state is failed' }); + }); + + await expectAccessApiToThrow({ + objects, + map, + counter, + errorMsg: 'failed as channel state is failed', + }); + await expectWriteApiToThrow({ objects, map, counter, errorMsg: 'failed as channel state is failed' }); + }, + }, + + { + description: 'public write API throws invalid channel state error when channel SUSPENDED', + action: async (ctx) => { + const { objects, channel, map, counter, helper } = ctx; + + // obtain batch context with valid channel state first + await objects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate channel state change + helper.recordPrivateApi('call.channel.requestState'); + channel.requestState('suspended'); + + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: 'failed as channel state is suspended' }); + }); + + await expectWriteApiToThrow({ + objects, + map, + counter, + errorMsg: 'failed as channel state is suspended', + }); + }, + }, + + { + description: 'public write API throws invalid channel option when "echoMessages" is disabled', + action: async (ctx) => { + const { objects, client, map, counter, helper } = ctx; + + // obtain batch context with valid client options first + await objects.batch((ctx) => { + const map = ctx.getRoot().get('map'); + const counter = ctx.getRoot().get('counter'); + // now simulate echoMessages was disabled + helper.recordPrivateApi('write.realtime.options.echoMessages'); + client.options.echoMessages = false; + + expectWriteBatchApiToThrow({ ctx, map, counter, errorMsg: '"echoMessages" client option' }); + }); + + await expectWriteApiToThrow({ objects, map, counter, errorMsg: '"echoMessages" client option' }); + }, + }, + ]; + + /** @nospec */ + forScenarios(this, clientConfigurationScenarios, async function (helper, scenario, clientOptions, channelName) { + const objectsHelper = new ObjectsHelper(helper); + const client = RealtimeWithObjects(helper, clientOptions); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + // attach with correct channel modes so we can create Objects on the root for testing. + // some scenarios will modify the underlying modes array to test specific behavior + const channel = client.channels.get(channelName, channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + const objectsCreatedPromise = Promise.all([ + waitForMapKeyUpdate(root, 'map'), + waitForMapKeyUpdate(root, 'counter'), + ]); + const map = await objects.createMap(); + const counter = await objects.createCounter(); + await root.set('map', map); + await root.set('counter', counter); + await objectsCreatedPromise; + + await scenario.action({ objects, objectsHelper, channelName, channel, root, map, counter, helper, client }); + }, client); + }); + + /** + * @spec TO3l8 + * @spec RSL1i + */ + it('object message publish respects connectionDetails.maxMessageSize', async function () { + const helper = this.test.helper; + const client = RealtimeWithObjects(helper, { clientId: 'test' }); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + await client.connection.once('connected'); + + const connectionManager = client.connection.connectionManager; + const connectionDetails = connectionManager.connectionDetails; + const connectionDetailsPromise = connectionManager.once('connectiondetails'); + + helper.recordPrivateApi('write.connectionManager.connectionDetails.maxMessageSize'); + connectionDetails.maxMessageSize = 64; + + helper.recordPrivateApi('call.connectionManager.activeProtocol.getTransport'); + helper.recordPrivateApi('call.transport.onProtocolMessage'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); + // forge lower maxMessageSize + connectionManager.activeProtocol.getTransport().onProtocolMessage( + createPM({ + action: 4, // CONNECTED + connectionDetails, + }), + ); + + helper.recordPrivateApi('listen.connectionManager.connectiondetails'); + await connectionDetailsPromise; + + const channel = client.channels.get('channel', channelOptionsWithObjects()); + const objects = channel.objects; + + await channel.attach(); + const root = await objects.getRoot(); + + const data = new Array(100).fill('a').join(''); + const error = await expectToThrowAsync( + async () => root.set('key', data), + 'Maximum size of object messages that can be published at once exceeded', + ); + + expect(error.code).to.equal(40009, 'Check maximum size of messages error has correct error code'); + }, client); + }); + + describe('ObjectMessage message size', () => { + const objectMessageSizeScenarios = [ + { + description: 'client id', + message: objectMessageFromValues({ + clientId: 'my-client', + }), + expected: Utils.dataSizeBytes('my-client'), + }, + { + description: 'extras', + message: objectMessageFromValues({ + extras: { foo: 'bar' }, + }), + expected: Utils.dataSizeBytes('{"foo":"bar"}'), + }, + { + description: 'object id', + message: objectMessageFromValues({ + operation: { objectId: 'object-id' }, + }), + expected: 0, + }, + { + description: 'nonce', + message: objectMessageFromValues({ + operation: { nonce: '1234567890' }, + }), + expected: 0, + }, + { + description: 'initial value', + message: objectMessageFromValues({ + operation: { + initialValue: BufferUtils.utf8Encode('{"counter":{"count":1}}'), + initialValueEncoding: 'json', + }, + }), + expected: 0, + }, + { + description: 'map create op no payload', + message: objectMessageFromValues({ + operation: { action: 0, objectId: 'object-id' }, + }), + expected: 0, + }, + { + description: 'map create op with object payload', + message: objectMessageFromValues( + { + operation: { + action: 0, + objectId: 'object-id', + map: { + semantics: 0, + entries: { 'key-1': { tombstone: false, data: { objectId: 'another-object-id' } } }, + }, + }, + }, + MessageEncoding, + ), + expected: Utils.dataSizeBytes('key-1'), + }, + { + description: 'map create op with string payload', + message: objectMessageFromValues( + { + operation: { + action: 0, + objectId: 'object-id', + map: { semantics: 0, entries: { 'key-1': { tombstone: false, data: { string: 'a string' } } } }, + }, + }, + MessageEncoding, + ), + expected: Utils.dataSizeBytes('key-1') + Utils.dataSizeBytes('a string'), + }, + { + description: 'map create op with bytes payload', + message: objectMessageFromValues( + { + operation: { + action: 0, + objectId: 'object-id', + map: { + semantics: 0, + entries: { 'key-1': { tombstone: false, data: { bytes: BufferUtils.utf8Encode('my-value') } } }, + }, + }, + }, + MessageEncoding, + ), + expected: Utils.dataSizeBytes('key-1') + Utils.dataSizeBytes(BufferUtils.utf8Encode('my-value')), + }, + { + description: 'map create op with boolean payload', + message: objectMessageFromValues( + { + operation: { + action: 0, + objectId: 'object-id', + map: { + semantics: 0, + entries: { + 'key-1': { tombstone: false, data: { boolean: true } }, + 'key-2': { tombstone: false, data: { boolean: false } }, + }, + }, + }, + }, + MessageEncoding, + ), + expected: Utils.dataSizeBytes('key-1') + Utils.dataSizeBytes('key-2') + 2, + }, + { + description: 'map create op with double payload', + message: objectMessageFromValues( + { + operation: { + action: 0, + objectId: 'object-id', + map: { + semantics: 0, + entries: { + 'key-1': { tombstone: false, data: { number: 123.456 } }, + 'key-2': { tombstone: false, data: { number: 0 } }, + }, + }, + }, + }, + MessageEncoding, + ), + expected: Utils.dataSizeBytes('key-1') + Utils.dataSizeBytes('key-2') + 16, + }, + { + description: 'map remove op', + message: objectMessageFromValues({ + operation: { action: 2, objectId: 'object-id', mapOp: { key: 'my-key' } }, + }), + expected: Utils.dataSizeBytes('my-key'), + }, + { + description: 'map set operation value=object', + message: objectMessageFromValues({ + operation: { + action: 1, + objectId: 'object-id', + mapOp: { key: 'my-key', data: { objectId: 'another-object-id' } }, + }, + }), + expected: Utils.dataSizeBytes('my-key'), + }, + { + description: 'map set operation value=string', + message: objectMessageFromValues({ + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { string: 'my-value' } } }, + }), + expected: Utils.dataSizeBytes('my-key') + Utils.dataSizeBytes('my-value'), + }, + { + description: 'map set operation value=bytes', + message: objectMessageFromValues({ + operation: { + action: 1, + objectId: 'object-id', + mapOp: { key: 'my-key', data: { bytes: BufferUtils.utf8Encode('my-value') } }, + }, + }), + expected: Utils.dataSizeBytes('my-key') + Utils.dataSizeBytes(BufferUtils.utf8Encode('my-value')), + }, + { + description: 'map set operation value=boolean true', + message: objectMessageFromValues({ + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { boolean: true } } }, + }), + expected: Utils.dataSizeBytes('my-key') + 1, + }, + { + description: 'map set operation value=boolean false', + message: objectMessageFromValues({ + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { boolean: false } } }, + }), + expected: Utils.dataSizeBytes('my-key') + 1, + }, + { + description: 'map set operation value=double', + message: objectMessageFromValues({ + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { number: 123.456 } } }, + }), + expected: Utils.dataSizeBytes('my-key') + 8, + }, + { + description: 'map set operation value=double 0', + message: objectMessageFromValues({ + operation: { action: 1, objectId: 'object-id', mapOp: { key: 'my-key', data: { number: 0 } } }, + }), + expected: Utils.dataSizeBytes('my-key') + 8, + }, + { + description: 'map object', + message: objectMessageFromValues({ + object: { + objectId: 'object-id', + map: { + semantics: 0, + entries: { + 'key-1': { tombstone: false, data: { string: 'a string' } }, + 'key-2': { tombstone: true, data: { string: 'another string' } }, + }, + }, + createOp: { + action: 0, + objectId: 'object-id', + map: { semantics: 0, entries: { 'key-3': { tombstone: false, data: { string: 'third string' } } } }, + }, + siteTimeserials: { aaa: lexicoTimeserial('aaa', 111, 111, 1) }, // shouldn't be counted + tombstone: false, + }, + }), + expected: + Utils.dataSizeBytes('key-1') + + Utils.dataSizeBytes('a string') + + Utils.dataSizeBytes('key-2') + + Utils.dataSizeBytes('another string') + + Utils.dataSizeBytes('key-3') + + Utils.dataSizeBytes('third string'), + }, + { + description: 'counter create op no payload', + message: objectMessageFromValues({ + operation: { action: 3, objectId: 'object-id' }, + }), + expected: 0, + }, + { + description: 'counter create op with payload', + message: objectMessageFromValues({ + operation: { action: 3, objectId: 'object-id', counter: { count: 1234567 } }, + }), + expected: 8, + }, + { + description: 'counter inc op', + message: objectMessageFromValues({ + operation: { action: 4, objectId: 'object-id', counterOp: { amount: 123.456 } }, + }), + expected: 8, + }, + { + description: 'counter object', + message: objectMessageFromValues({ + object: { + objectId: 'object-id', + counter: { count: 1234567 }, + createOp: { + action: 3, + objectId: 'object-id', + counter: { count: 9876543 }, + }, + siteTimeserials: { aaa: lexicoTimeserial('aaa', 111, 111, 1) }, // shouldn't be counted + tombstone: false, + }, + }), + expected: 8 + 8, + }, + ]; + + /** @nospec */ + forScenarios(this, objectMessageSizeScenarios, function (helper, scenario) { + const client = RealtimeWithObjects(helper, { autoConnect: false }); + helper.recordPrivateApi('call.ObjectMessage.encode'); + ObjectsPlugin.ObjectMessage.encode(scenario.message, client); + helper.recordPrivateApi('call.BufferUtils.utf8Encode'); // was called by a scenario to create buffers + helper.recordPrivateApi('call.ObjectMessage.fromValues'); // was called by a scenario to create an ObjectMessage instance + helper.recordPrivateApi('call.Utils.dataSizeBytes'); // was called by a scenario to calculated the expected byte size + helper.recordPrivateApi('call.ObjectMessage.getMessageSize'); + expect(scenario.message.getMessageSize()).to.equal(scenario.expected); + }); + }); + }); + + /** @nospec */ + it('can attach to channel with object modes', async function () { + const helper = this.test.helper; + const client = helper.AblyRealtime(); + + await helper.monitorConnectionThenCloseAndFinishAsync(async () => { + const objectsModes = ['object_subscribe', 'object_publish']; + const channelOptions = { modes: objectsModes }; + const channel = client.channels.get('channel', channelOptions); + + await channel.attach(); + + helper.recordPrivateApi('read.channel.channelOptions'); + expect(channel.channelOptions).to.deep.equal(channelOptions, 'Check expected channel options'); + expect(channel.modes).to.deep.equal(objectsModes, 'Check expected modes'); + }, client); + }); + }); +}); diff --git a/test/realtime/presence.test.js b/test/realtime/presence.test.js index c6f4e441fa..6bf79d78d3 100644 --- a/test/realtime/presence.test.js +++ b/test/realtime/presence.test.js @@ -2,7 +2,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async, chai) { const { expect, assert } = chai; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); var PresenceMessage = Ably.Realtime.PresenceMessage; function extractClientIds(presenceSet) { @@ -2093,7 +2093,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async cb(); }); /* Inject an ATTACHED with RESUMED and HAS_PRESENCE both false */ - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); channel.processMessage( createPM({ action: 11, diff --git a/test/realtime/reauth.test.js b/test/realtime/reauth.test.js index 0653bdc3e7..081b3cd2b5 100644 --- a/test/realtime/reauth.test.js +++ b/test/realtime/reauth.test.js @@ -178,7 +178,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { } function testCase(thisInDescribe, name, createSteps) { - Helper.testOnAllTransports(thisInDescribe, name, function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(thisInDescribe, name, function (realtimeOpts) { return function (done) { const helper = this.test.helper; var _steps = createSteps(helper).slice(); diff --git a/test/realtime/resume.test.js b/test/realtime/resume.test.js index 0ae4554dd3..7fd47def5a 100644 --- a/test/realtime/resume.test.js +++ b/test/realtime/resume.test.js @@ -140,7 +140,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * Related to RTN15b, RTN15c. * @nospec */ - Helper.testOnAllTransports(this, 'resume_inactive', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'resume_inactive', function (realtimeOpts) { return function (done) { resume_inactive(done, this.test.helper, 'resume_inactive' + String(Math.random()), {}, realtimeOpts); }; @@ -266,7 +266,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * Related to RTN15b, RTN15c. * @nospec */ - Helper.testOnAllTransports(this, 'resume_active', function (realtimeOpts) { + Helper.testOnAllTransportsAndProtocols(this, 'resume_active', function (realtimeOpts) { return function (done) { resume_active(done, this.test.helper, 'resume_active' + String(Math.random()), {}, realtimeOpts); }; @@ -276,7 +276,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * Resume with loss of continuity * @spec RTN15c7 */ - Helper.testOnAllTransports( + Helper.testOnAllTransportsAndProtocols( this, 'resume_lost_continuity', function (realtimeOpts) { @@ -353,7 +353,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * Resume with token error * @spec RTN15c5 */ - Helper.testOnAllTransports( + Helper.testOnAllTransportsAndProtocols( this, 'resume_token_error', function (realtimeOpts) { @@ -411,7 +411,7 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * Resume with fatal error * @spec RTN15c4 */ - Helper.testOnAllTransports( + Helper.testOnAllTransportsAndProtocols( this, 'resume_fatal_error', function (realtimeOpts) { diff --git a/test/realtime/sync.test.js b/test/realtime/sync.test.js index 40437ed845..b4cd413fb4 100644 --- a/test/realtime/sync.test.js +++ b/test/realtime/sync.test.js @@ -2,7 +2,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async, chai) { var expect = chai.expect; - var createPM = Ably.protocolMessageFromDeserialized; + var createPM = Ably.makeProtocolMessageFromDeserialized(); describe('realtime/sync', function () { this.timeout(60 * 1000); @@ -50,7 +50,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channelName = 'syncexistingset', channel = realtime.channels.get(channelName); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.channel.processMessage'); await channel.processMessage( createPM({ @@ -183,7 +183,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channelName = 'sync_member_arrives_in_middle', channel = realtime.channels.get(channelName); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.channel.processMessage'); await channel.processMessage( createPM({ @@ -303,7 +303,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channelName = 'sync_member_arrives_normally_after_came_in_sync', channel = realtime.channels.get(channelName); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.channel.processMessage'); await channel.processMessage( createPM({ @@ -401,7 +401,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channelName = 'sync_member_arrives_normally_before_comes_in_sync', channel = realtime.channels.get(channelName); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.channel.processMessage'); await channel.processMessage( createPM({ @@ -503,7 +503,7 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async channelName = 'sync_ordering', channel = realtime.channels.get(channelName); - helper.recordPrivateApi('call.protocolMessageFromDeserialized'); + helper.recordPrivateApi('call.makeProtocolMessageFromDeserialized'); helper.recordPrivateApi('call.channel.processMessage'); await channel.processMessage( createPM({ diff --git a/test/rest/history.test.js b/test/rest/history.test.js index bc26ebbc0f..ce58245969 100644 --- a/test/rest/history.test.js +++ b/test/rest/history.test.js @@ -31,7 +31,8 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * @spec RSL2 * @spec RSL2a */ - Helper.restTestOnJsonMsgpack('history_simple', async function (rest, channelName, helper) { + Helper.testOnJsonMsgpack('history_simple', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); var testchannel = rest.channels.get('persisted:' + channelName); /* first, send a number of events to this channel */ @@ -63,7 +64,8 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * @spec RSL2 * @spec RSL2a */ - Helper.restTestOnJsonMsgpack('history_multiple', async function (rest, channelName, helper) { + Helper.testOnJsonMsgpack('history_multiple', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); var testchannel = rest.channels.get('persisted:' + channelName); /* first, send a number of events to this channel */ @@ -92,7 +94,8 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { * @spec RSL2b2 * @specpartial RSL2b3 - should also test maximum supported limit of 1000 */ - Helper.restTestOnJsonMsgpack('history_simple_paginated_b', async function (rest, channelName, helper) { + Helper.testOnJsonMsgpack('history_simple_paginated_b', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); var testchannel = rest.channels.get('persisted:' + channelName); /* first, send a number of events to this channel */ @@ -259,7 +262,8 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { }); /** @nospec */ - Helper.restTestOnJsonMsgpack('history_encoding_errors', async function (rest, channelName) { + Helper.testOnJsonMsgpack('history_encoding_errors', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); var testchannel = rest.channels.get('persisted:' + channelName); var badMessage = { name: 'jsonUtf8string', encoding: 'json/utf-8', data: '{"foo":"bar"}' }; testchannel.publish(badMessage); @@ -272,7 +276,8 @@ define(['shared_helper', 'async', 'chai'], function (Helper, async, chai) { }); /** @specpartial TG4 - in the context of RestChannel#history */ - Helper.restTestOnJsonMsgpack('history_no_next_page', async function (rest, channelName) { + Helper.testOnJsonMsgpack('history_no_next_page', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); const channel = rest.channels.get(channelName); const firstPage = await channel.history(); diff --git a/test/rest/request.test.js b/test/rest/request.test.js index 7132b0c5ef..c54a7f7430 100644 --- a/test/rest/request.test.js +++ b/test/rest/request.test.js @@ -28,7 +28,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @specpartial RSC7e - tests providing a version value in .request parameters * @specpartial CSV2c - tests version is provided in http requests */ - Helper.restTestOnJsonMsgpack('request_version', function (rest, _, helper) { + Helper.testOnJsonMsgpack('request_version', function (options, _, helper) { + const rest = helper.AblyRest(options); const version = 150; // arbitrarily chosen async function testRequestHandler(_, __, headers) { @@ -56,7 +57,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec HP5 * @specpartial RSC19f - basic test for passing a http method, path and version parameters */ - Helper.restTestOnJsonMsgpack('request_time', async function (rest) { + Helper.testOnJsonMsgpack('request_time', async function (options, _, helper) { + const rest = helper.AblyRest(options); const res = await rest.request('get', '/time', 3, null, null, null); expect(res.statusCode).to.equal(200, 'Check statusCode'); expect(res.success).to.equal(true, 'Check success'); @@ -72,7 +74,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec HP6 * @spec HP7 */ - Helper.restTestOnJsonMsgpack('request_404', async function (rest) { + Helper.testOnJsonMsgpack('request_404', async function (options, _, helper) { + const rest = helper.AblyRest(options); /* NB: can't just use /invalid or something as the CORS preflight will * fail. Need something superficially a valid path but where the actual * request fails */ @@ -110,7 +113,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @specpartial HP2 - tests overriden .next method only * @specpartial RSC19f - more tests with passing other methods, body and parameters */ - Helper.restTestOnJsonMsgpack('request_post_get_messages', async function (rest, channelName) { + Helper.testOnJsonMsgpack('request_post_get_messages', async function (options, channelName, helper) { + const rest = helper.AblyRest(options); var channelPath = '/channels/' + channelName + '/messages', msgone = { name: 'faye', data: 'whittaker' }, msgtwo = { name: 'martin', data: 'reed' }; @@ -155,7 +159,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @spec HP7 * @specpartial RSC19f - more tests with POST method and passing body */ - Helper.restTestOnJsonMsgpack('request_batch_api_success', async function (rest, name) { + Helper.testOnJsonMsgpack('request_batch_api_success', async function (options, name, helper) { + const rest = helper.AblyRest(options); var body = { channels: [name + '1', name + '2'], messages: { data: 'foo' } }; const res = await rest.request('POST', '/messages', 2, {}, body, {}); @@ -186,7 +191,8 @@ define(['ably', 'shared_helper', 'async', 'chai'], function (Ably, Helper, async * @specpartial RSC19f - more tests with POST method and passing body * @specskip */ - Helper.restTestOnJsonMsgpack.skip('request_batch_api_partial_success', async function (rest, name) { + Helper.testOnJsonMsgpack.skip('request_batch_api_partial_success', async function (options, name, helper) { + const rest = helper.AblyRest(options); var body = { channels: [name, '[invalid', ''], messages: { data: 'foo' } }; var res = await rest.request('POST', '/messages', 2, {}, body, {}); diff --git a/test/rest/status.test.js b/test/rest/status.test.js index c1e52a20b4..d138af6e6c 100644 --- a/test/rest/status.test.js +++ b/test/rest/status.test.js @@ -34,7 +34,8 @@ define(['shared_helper', 'chai'], function (Helper, chai) { * @spec CHM2e * @spec CHM2f */ - Helper.restTestOnJsonMsgpack('status0', async function (rest) { + Helper.testOnJsonMsgpack('status0', async function (options, _, helper) { + const rest = helper.AblyRest(options); var channel = rest.channels.get('status0'); var channelDetails = await channel.status(); expect(channelDetails.channelId).to.equal('status0'); diff --git a/test/support/browser_file_list.js b/test/support/browser_file_list.js index a35ea5b6c9..c6cd5668c5 100644 --- a/test/support/browser_file_list.js +++ b/test/support/browser_file_list.js @@ -39,6 +39,7 @@ window.__testFiles__.files = { 'test/realtime/failure.test.js': true, 'test/realtime/history.test.js': true, 'test/realtime/init.test.js': true, + 'test/realtime/objects.test.js': true, 'test/realtime/message.test.js': true, 'test/realtime/presence.test.js': true, 'test/realtime/reauth.test.js': true, diff --git a/typedoc.json b/typedoc.json index 24faf3bad9..bdbe58b4dc 100644 --- a/typedoc.json +++ b/typedoc.json @@ -20,5 +20,6 @@ "TypeAlias", "Variable", "Namespace" - ] + ], + "intentionallyNotExported": ["__global.AblyObjectsTypes"] }