From 617acef96f2b4e2fd2e9c173d5c720aded9f42db Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Tue, 17 Dec 2024 16:04:43 +0100 Subject: [PATCH 1/8] feat: Do not hydrate doc if the relationship does not exist We used to hydrate any document with the relationships existing in the provided schema, even though the relationship does not exist on the document. We now hydrate only if the relationship is set in the document. Note we can still force the hydratation through a `forceHydration` option, to ease migrations on apps with many relations. BREAKING CHANGE: the relationship hydration is made only if the relationship exists in the document, so the developer should not assume a `document.relationshipName` is always defined, anymore. As an alternative, it is now possible to pass `forceHydration` on cozy-client options to ease migration. However, please not this has performance impact, as it forces extra-check on store queries evaluation. --- packages/cozy-client/src/CozyClient.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/cozy-client/src/CozyClient.js b/packages/cozy-client/src/CozyClient.js index ef98bc1365..b9106a81a7 100644 --- a/packages/cozy-client/src/CozyClient.js +++ b/packages/cozy-client/src/CozyClient.js @@ -121,6 +121,7 @@ const DOC_UPDATE = 'update' * @property {import("./types").ClientCapabilities} [capabilities] - Capabilities sent by the stack * @property {boolean} [store] - If set to false, the client will not instantiate a Redux store automatically. Use this if you want to merge cozy-client's store with your own redux store. See [here](https://docs.cozy.io/en/cozy-client/react-integration/#1b-use-your-own-redux-store) for more information. * @property {import('./performances/types').PerformanceAPI} [performanceApi] - The performance API that can be used to measure performances + * @property {boolean} [forceHydratation] - If set to true, all documents will be hydrated w.r.t. the provided schema's relationships, even if the relationship does not exist on the doc. */ /** @@ -1332,9 +1333,11 @@ client.query(Q('io.cozy.bills'))`) hydrateRelationships(document, schemaRelationships) { const methods = this.getRelationshipStoreAccessors() - return mapValues(schemaRelationships, (assoc, name) => - createAssociation(document, assoc, methods) - ) + return mapValues(schemaRelationships, (assoc, name) => { + if (this.options?.forceHydratation || document.relationships?.[assoc]) { + return createAssociation(document, assoc, methods) + } + }) } /** From 5fb23bb747ea12fb4dd767b2c0aa54d7d69b3a6d Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Tue, 17 Dec 2024 16:13:46 +0100 Subject: [PATCH 2/8] fix: Correctly fetch documents relationships --- packages/cozy-client/src/CozyClient.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cozy-client/src/CozyClient.js b/packages/cozy-client/src/CozyClient.js index b9106a81a7..cfeece19c4 100644 --- a/packages/cozy-client/src/CozyClient.js +++ b/packages/cozy-client/src/CozyClient.js @@ -1217,7 +1217,7 @@ client.query(Q('io.cozy.bills'))`) if (queryDef instanceof QueryDefinition) { definitions.push(queryDef) } else { - documents.push(queryDef) + documents.push(doc) } } catch { // eslint-disable-next-line From df2b76d60e9eb2c12947a0e6eb05479596680c0a Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Tue, 17 Dec 2024 16:20:02 +0100 Subject: [PATCH 3/8] feat: Improve store evaluation --- docs/api/cozy-client/README.md | 4 +- docs/api/cozy-client/classes/CozyClient.md | 1 + packages/cozy-client/src/CozyClient.js | 20 +- packages/cozy-client/src/hooks/useQuery.js | 3 +- packages/cozy-client/src/hooks/utils.js | 107 +++++++++ packages/cozy-client/src/hooks/utils.spec.js | 220 +++++++++++++++++++ packages/cozy-client/src/store/documents.js | 1 - packages/cozy-client/src/types.js | 9 + packages/cozy-client/types/CozyClient.d.ts | 9 + packages/cozy-client/types/hooks/utils.d.ts | 1 + packages/cozy-client/types/types.d.ts | 22 ++ 11 files changed, 391 insertions(+), 6 deletions(-) create mode 100644 packages/cozy-client/src/hooks/utils.js create mode 100644 packages/cozy-client/src/hooks/utils.spec.js create mode 100644 packages/cozy-client/types/hooks/utils.d.ts diff --git a/docs/api/cozy-client/README.md b/docs/api/cozy-client/README.md index 7950db15f0..550eac9db9 100644 --- a/docs/api/cozy-client/README.md +++ b/docs/api/cozy-client/README.md @@ -976,7 +976,7 @@ Retrieve intance info like context, uuid, disk usage etc *Defined in* -[packages/cozy-client/src/hooks/useQuery.js:93](https://github.com/cozy/cozy-client/blob/master/packages/cozy-client/src/hooks/useQuery.js#L93) +[packages/cozy-client/src/hooks/useQuery.js:94](https://github.com/cozy/cozy-client/blob/master/packages/cozy-client/src/hooks/useQuery.js#L94) *** @@ -999,7 +999,7 @@ Fetches a queryDefinition and returns the queryState *Defined in* -[packages/cozy-client/src/hooks/useQuery.js:28](https://github.com/cozy/cozy-client/blob/master/packages/cozy-client/src/hooks/useQuery.js#L28) +[packages/cozy-client/src/hooks/useQuery.js:29](https://github.com/cozy/cozy-client/blob/master/packages/cozy-client/src/hooks/useQuery.js#L29) *** diff --git a/docs/api/cozy-client/classes/CozyClient.md b/docs/api/cozy-client/classes/CozyClient.md index 75c391d847..6b9da2e9d8 100644 --- a/docs/api/cozy-client/classes/CozyClient.md +++ b/docs/api/cozy-client/classes/CozyClient.md @@ -148,6 +148,7 @@ Cozy-Client will automatically call `this.login()` if provided with a token and | `autoHydrate` | `boolean` | - | | `backgroundFetching` | `boolean` | If set to true, backgroundFetching will be enabled by default on every query. Meaning that, when the fetchStatus has already been loaded, it won't be updated during future fetches. Instead, a `isFetching` attribute will be used to indicate when background fetching is started. | | `client` | `any` | - | +| `forceHydratation` | `boolean` | If set to true, all documents will be hydrated w.r.t. the provided schema's relationships, even if the relationship does not exist on the doc. | | `oauth` | `any` | - | | `onError` | `Function` | Default callback if a query is errored | | `onTokenRefresh` | `Function` | - | diff --git a/packages/cozy-client/src/CozyClient.js b/packages/cozy-client/src/CozyClient.js index cfeece19c4..ffbcb6b087 100644 --- a/packages/cozy-client/src/CozyClient.js +++ b/packages/cozy-client/src/CozyClient.js @@ -1447,13 +1447,29 @@ client.query(Q('io.cozy.bills'))`) return queryResults } - const data = + const hydratedData = hydrated && doctype ? this.hydrateDocuments(doctype, queryResults.data) : queryResults.data + + const relationships = this.schema.getDoctypeSchema(doctype)?.relationships + const relationshipNames = relationships + ? Object.keys(relationships) + : null + + // The `data` array contains the hydrated data with the relationships, if any. + // The `storeData` array contains the documents from the store: this is useful to preserve + // referential equality, to be later evaluated to determine whether or not the + // documents had changed. return { ...queryResults, - data: isSingleDocQuery && singleDocData ? data[0] : data + data: + isSingleDocQuery && singleDocData ? hydratedData[0] : hydratedData, + storeData: + isSingleDocQuery && singleDocData + ? queryResults.data[0] + : queryResults.data, + relationshipNames } } catch (e) { logger.warn( diff --git a/packages/cozy-client/src/hooks/useQuery.js b/packages/cozy-client/src/hooks/useQuery.js index 7bff34c242..1e104afcc9 100644 --- a/packages/cozy-client/src/hooks/useQuery.js +++ b/packages/cozy-client/src/hooks/useQuery.js @@ -5,6 +5,7 @@ import useClient from './useClient' import logger from '../logger' import { clientContext } from '../context' import { QueryDefinition } from '../queries/dsl' +import { equalityCheckForQuery } from './utils' const useSelector = createSelectorHook(clientContext) @@ -61,7 +62,7 @@ const useQuery = (queryDefinition, options) => { hydrated: get(options, 'hydrated', true), singleDocData: get(options, 'singleDocData', false) }) - }) + }, equalityCheckForQuery) useEffect( () => { diff --git a/packages/cozy-client/src/hooks/utils.js b/packages/cozy-client/src/hooks/utils.js new file mode 100644 index 0000000000..8904f6997f --- /dev/null +++ b/packages/cozy-client/src/hooks/utils.js @@ -0,0 +1,107 @@ +/** + * Equality check + * + * Note we do not make a shallow equality check on documents, as it is less efficient and should + * not be necessary: the queryResult.data is built by extracting documents from the state, thus + * preserving references. + * + * @param {import("../types").QueryStateResult} queryResA - A query result to compare + * @param {import("../types").QueryStateResult} queryResB - A query result to compare + * @returns + */ +export const equalityCheckForQuery = (queryResA, queryResB) => { + //console.log('Call equality check : ', queryResA, queryResB) + if (queryResA === queryResB) { + // Referential equality + return true + } + + if ( + typeof queryResA !== 'object' || + queryResA === null || + typeof queryResB !== 'object' || + queryResB === null + ) { + // queryResA or queryResB is not an object or null + return false + } + + if (queryResA.id !== queryResB.id) { + return false + } + if (queryResA.fetchStatus !== queryResB.fetchStatus) { + return false + } + + const docsA = queryResA.storeData + const docsB = queryResB.storeData + if (!docsA || !docsB) { + // No data to check + return false + } + if (!Array.isArray(docsA) && !Array.isArray(docsB) && docsA !== docsB) { + // Only one doc + return false + } + + if ( + Array.isArray(docsA) && + Array.isArray(docsB) && + !arraysHaveSameLength(docsA, docsB) + ) { + // A document was added or removed + return false + } + + if (Array.isArray(docsA) && Array.isArray(docsB)) { + for (let i = 0; i < docsA.length; i++) { + if (docsA[i] !== docsB[i]) { + // References should be the same for non-updated documents + return false + } + } + } + + if (queryResA.relationshipNames) { + // In case of relationships, we cannot check referential equality, because we + // "hydrate" the data by creating a new instance of the related relationship class. + // Thus, we check the document revision instead. + const hydratedDataA = queryResA.data + const hydratedDataB = queryResB.data + if (!Array.isArray(hydratedDataA) && !Array.isArray(hydratedDataB)) { + // One doc with changed relationship + return revsAreEqual(hydratedDataA, hydratedDataB) + } + if (!arraysHaveSameLength(hydratedDataA, hydratedDataB)) { + // A relationship have been added or removed + return false + } + if (Array.isArray(hydratedDataA) && Array.isArray(hydratedDataB)) { + for (let i = 0; i < hydratedDataA.length; i++) { + for (const name of queryResA.relationshipNames) { + // Check hydrated relationship + const includedA = hydratedDataA[i][name] + const includedB = hydratedDataB[i][name] + if (includedA && includedB) { + if (!revsAreEqual(includedA, includedB)) { + return false + } + } + } + } + } + } + return true +} + +const revsAreEqual = (docA, docB) => { + return docA?._rev === docB?._rev +} + +const arraysHaveSameLength = (arrayA, arrayB) => { + return ( + Array.isArray(arrayA) && + Array.isArray(arrayB) && + arrayA.length === arrayB.length + ) +} diff --git a/packages/cozy-client/src/hooks/utils.spec.js b/packages/cozy-client/src/hooks/utils.spec.js new file mode 100644 index 0000000000..023d48bebf --- /dev/null +++ b/packages/cozy-client/src/hooks/utils.spec.js @@ -0,0 +1,220 @@ +import { equalityCheckForQuery } from './utils' + +const mapIdsToDocuments = (state, doctype, ids) => { + return ids.map(id => state[doctype][id]) +} + +const state = { + documents: { + 'io.cozy.files': { + doc1: { + _id: 'doc1' + }, + doc2: { + _id: 'doc2' + }, + doc3: { + _id: 'doc3' + } + } + }, + queries: { + query1: { + id: 'query1', + data: ['doc1', 'doc2'] + }, + query2: { + id: 'query2', + data: ['doc2'] + } + } +} + +const defaultQueryResult = { + id: 1, + data: [], + fetchStatus: 'loaded', + relationshipNames: null +} + +describe('equalityCheckForQuery', () => { + const queryResultA1 = { + id: 1, + storeData: mapIdsToDocuments(state.documents, 'io.cozy.files', [ + 'doc1', + 'doc2' + ]), + ...defaultQueryResult + } + const queryResultA2 = { + id: 1, + storeData: mapIdsToDocuments(state.documents, 'io.cozy.files', [ + 'doc1', + 'doc2' + ]), + ...defaultQueryResult + } + const queryResultA3 = { + id: 1, + storeData: mapIdsToDocuments(state.documents, 'io.cozy.files', [ + 'doc1', + 'doc2', + 'doc3' + ]), + ...defaultQueryResult + } + const queryResultA4 = { + id: 1, + storeData: mapIdsToDocuments(state.documents, 'io.cozy.files', [ + 'doc2', + 'doc3' + ]), + ...defaultQueryResult + } + const queryResultB1 = { + id: 2, + storeData: mapIdsToDocuments(state.documents, 'io.cozy.files', ['doc2']), + ...defaultQueryResult + } + + const queryResultB2 = { + id: 2, + storeData: mapIdsToDocuments(state.documents, 'io.cozy.files', ['doc3']), + ...defaultQueryResult + } + + const queryResultC1 = { + id: 3, + storeData: state.documents['io.cozy.files'].doc1, + data: {}, + ...defaultQueryResult + } + + const queryResultC2 = { + id: 3, + storeData: state.documents['io.cozy.files'].doc1, + data: {}, + ...defaultQueryResult + } + + const queryResultC3 = { + id: 3, + storeData: state.documents['io.cozy.files'].doc2, + data: {}, + ...defaultQueryResult + } + + it('should return true for referential equality', () => { + expect(equalityCheckForQuery(queryResultA1, queryResultA1)).toBe(true) + expect(equalityCheckForQuery(null, null)).toBe(true) + }) + + it('should return false if one object is null', () => { + expect(equalityCheckForQuery(null, queryResultA1)).toBe(false) + expect(equalityCheckForQuery(queryResultA1, null)).toBe(false) + }) + + it('should return false if one or both objects are not objects', () => { + // @ts-ignore + expect(equalityCheckForQuery('notAnObject', queryResultA1)).toBe(false) + // @ts-ignore + expect(equalityCheckForQuery(queryResultA1, 'notAnObject')).toBe(false) + }) + + it('should return false if `id` properties are different', () => { + expect(equalityCheckForQuery(queryResultA1, queryResultB1)).toBe(false) + }) + + it('should return false if one or both objects lack `data`', () => { + // @ts-ignore + expect(equalityCheckForQuery({ id: 1 }, queryResultA1)).toBe(false) + // @ts-ignore + expect(equalityCheckForQuery(queryResultA1, { id: 1 })).toBe(false) + }) + + it('should return false if `data` lengths are different', () => { + expect(equalityCheckForQuery(queryResultA1, queryResultA3)).toBe(false) + }) + + it('should return false if elements in `data` are different', () => { + expect(equalityCheckForQuery(queryResultA1, queryResultA3)).toBe(false) + expect(equalityCheckForQuery(queryResultA3, queryResultA4)).toBe(false) + expect(equalityCheckForQuery(queryResultB1, queryResultB2)).toBe(false) + }) + + it('should return true for matching data array, with equal references ', () => { + expect(equalityCheckForQuery(queryResultA1, queryResultA2)).toBe(true) + }) + + it('should return false for matching data array, with different references ', () => { + const queryResShallowCopyA1 = { + ...queryResultA1, + storeData: JSON.parse(JSON.stringify(queryResultA1.storeData)) // Deep copy + } + expect(equalityCheckForQuery(queryResultA1, queryResShallowCopyA1)).toBe( + false + ) + }) + + it('should return true for matching object data', () => { + expect(equalityCheckForQuery(queryResultC1, queryResultC2)).toBe(true) + }) + it('should return false for different object data', () => { + expect(equalityCheckForQuery(queryResultC1, queryResultC3)).toBe(false) + }) +}) + +describe('equalityCheckForQuery with relationships', () => { + const queryResA = { + ...defaultQueryResult, + relationshipNames: ['relation1'], + storeData: mapIdsToDocuments(state.documents, 'io.cozy.files', [ + 'doc1', + 'doc2' + ]), + data: [{ relation1: { _rev: 'rev1' } }] + } + + const queryResB = { + ...defaultQueryResult, + relationshipNames: ['relation1'], + storeData: mapIdsToDocuments(state.documents, 'io.cozy.files', [ + 'doc1', + 'doc2' + ]), + data: [{ relation1: { _rev: 'rev2' } }] + } + + const queryResC = { + ...defaultQueryResult, + relationshipNames: ['relation1'], + storeData: mapIdsToDocuments(state.documents, 'io.cozy.files', [ + 'doc1', + 'doc2' + ]), + data: [{ relation1: { _rev: 'rev1' } }, { relation1: { _rev: 'rev2' } }] + } + + const queryResD = { + ...defaultQueryResult, + relationshipNames: ['relation1', 'relation2'], + storeData: mapIdsToDocuments(state.documents, 'io.cozy.files', [ + 'doc1', + 'doc2' + ]), + data: [{ relation1: { _rev: 'rev1' } }, { relation2: { _rev: 'rev2' } }] + } + + it('returns true when data and relationship revisions match', () => { + expect(equalityCheckForQuery(queryResA, queryResA)).toBe(true) + expect(equalityCheckForQuery(queryResD, queryResD)).toBe(true) + }) + + it('returns false when relationship revisions differ', () => { + expect(equalityCheckForQuery(queryResA, queryResB)).toBe(false) + }) + + it('returns false when data lengths differ', () => { + expect(equalityCheckForQuery(queryResA, queryResC)).toBe(false) + }) +}) diff --git a/packages/cozy-client/src/store/documents.js b/packages/cozy-client/src/store/documents.js index 410ee7fe3a..8992b0d998 100644 --- a/packages/cozy-client/src/store/documents.js +++ b/packages/cozy-client/src/store/documents.js @@ -173,7 +173,6 @@ export const extractAndMergeDocument = (data, updatedStateWithIncluded) => { let mergedData = Object.assign({}, updatedStateWithIncluded) mergedData[doctype] = Object.assign({}, updatedStateWithIncluded[doctype]) - Object.values(sortedData).map(data => { const id = properId(data) if (mergedData[doctype][id]) { diff --git a/packages/cozy-client/src/types.js b/packages/cozy-client/src/types.js index 5e404c938f..6c0467e0ca 100644 --- a/packages/cozy-client/src/types.js +++ b/packages/cozy-client/src/types.js @@ -277,6 +277,15 @@ import { QueryDefinition } from './queries/dsl' * @property {object|Array} data */ +/** + * @typedef {object} QueryStateResult + * @property {object|Array} storeData - Collection of store's documents + * @property {object|Array} data - Collection of hydrated documents + * @property {Array} relationshipNames - The relationships names, used to check hydrated documents + * @property {string|number} id - The query id + * @property {string} fetchStatus - The query fetching status + */ + /** * @typedef {QueryStateWithoutData & QueryStateData} QueryState */ diff --git a/packages/cozy-client/types/CozyClient.d.ts b/packages/cozy-client/types/CozyClient.d.ts index c4080afb3a..d26855a346 100644 --- a/packages/cozy-client/types/CozyClient.d.ts +++ b/packages/cozy-client/types/CozyClient.d.ts @@ -80,6 +80,10 @@ export type ClientOptions = { * - The performance API that can be used to measure performances */ performanceApi?: import('./performances/types').PerformanceAPI; + /** + * - If set to true, all documents will be hydrated w.r.t. the provided schema's relationships, even if the relationship does not exist on the doc. + */ + forceHydratation?: boolean; }; /** * @typedef {import("./types").CozyClientDocument} CozyClientDocument @@ -104,6 +108,7 @@ export type ClientOptions = { * @property {import("./types").ClientCapabilities} [capabilities] - Capabilities sent by the stack * @property {boolean} [store] - If set to false, the client will not instantiate a Redux store automatically. Use this if you want to merge cozy-client's store with your own redux store. See [here](https://docs.cozy.io/en/cozy-client/react-integration/#1b-use-your-own-redux-store) for more information. * @property {import('./performances/types').PerformanceAPI} [performanceApi] - The performance API that can be used to measure performances + * @property {boolean} [forceHydratation] - If set to true, all documents will be hydrated w.r.t. the provided schema's relationships, even if the relationship does not exist on the doc. */ /** * Responsible for @@ -219,6 +224,10 @@ declare class CozyClient { * - If set to false, the client will not instantiate a Redux store automatically. Use this if you want to merge cozy-client's store with your own redux store. See [here](https://docs.cozy.io/en/cozy-client/react-integration/#1b-use-your-own-redux-store) for more information. */ store?: boolean; + /** + * - If set to true, all documents will be hydrated w.r.t. the provided schema's relationships, even if the relationship does not exist on the doc. + */ + forceHydratation?: boolean; }; queryIdGenerator: QueryIDGenerator; isLogged: boolean; diff --git a/packages/cozy-client/types/hooks/utils.d.ts b/packages/cozy-client/types/hooks/utils.d.ts new file mode 100644 index 0000000000..4cd1825eab --- /dev/null +++ b/packages/cozy-client/types/hooks/utils.d.ts @@ -0,0 +1 @@ +export function equalityCheckForQuery(queryResA: import("../types").QueryStateResult, queryResB: import("../types").QueryStateResult): boolean; diff --git a/packages/cozy-client/types/types.d.ts b/packages/cozy-client/types/types.d.ts index 3f713da2ae..fbc8bf702b 100644 --- a/packages/cozy-client/types/types.d.ts +++ b/packages/cozy-client/types/types.d.ts @@ -508,6 +508,28 @@ export type QueryStateWithoutData = { export type QueryStateData = { data: object | any[]; }; +export type QueryStateResult = { + /** + * - Collection of store's documents + */ + storeData: object | any[]; + /** + * - Collection of hydrated documents + */ + data: object | any[]; + /** + * - The relationships names, used to check hydrated documents + */ + relationshipNames: Array; + /** + * - The query id + */ + id: string | number; + /** + * - The query fetching status + */ + fetchStatus: string; +}; export type QueryState = QueryStateWithoutData & QueryStateData; export type AutoUpdateOptions = any; export type QueryOptions = { From a19c594323ecd01398958e5bfeb1fe0bc271f7b2 Mon Sep 17 00:00:00 2001 From: Ldoppea Date: Thu, 27 Feb 2025 18:52:18 +0100 Subject: [PATCH 4/8] feat: Improve persistVirtualDocuments performances --- packages/cozy-client/src/CozyClient.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/cozy-client/src/CozyClient.js b/packages/cozy-client/src/CozyClient.js index ffbcb6b087..bfc481d00a 100644 --- a/packages/cozy-client/src/CozyClient.js +++ b/packages/cozy-client/src/CozyClient.js @@ -1157,7 +1157,18 @@ client.query(Q('io.cozy.bills'))`) if (!Array.isArray(data)) { await this.persistVirtualDocument(data, enforce) } else { - for (const document of data) { + const documentsToPersist = data.filter(document => { + if (!document || document.cozyLocalOnly) { + return false + } + + if ((!document.meta?.rev && !document._rev) || enforce) { + return true + } + + return false + }) + for (const document of documentsToPersist) { await this.persistVirtualDocument(document, enforce) } } From 89645bb339c214eb4e888de41545feff7963fd85 Mon Sep 17 00:00:00 2001 From: Ldoppea Date: Fri, 28 Feb 2025 20:05:36 +0100 Subject: [PATCH 5/8] test: Add more unit tests for jsonapi --- packages/cozy-pouch-link/src/jsonapi.spec.js | 431 +++++++++++++++++++ 1 file changed, 431 insertions(+) diff --git a/packages/cozy-pouch-link/src/jsonapi.spec.js b/packages/cozy-pouch-link/src/jsonapi.spec.js index f507c1034b..eed86cc2b7 100644 --- a/packages/cozy-pouch-link/src/jsonapi.spec.js +++ b/packages/cozy-pouch-link/src/jsonapi.spec.js @@ -31,6 +31,7 @@ const DELETED_DOC_FIXTURE = { const token = 'fake_token' const uri = 'https://claude.mycozy.cloud' const client = new CozyClient({ token, uri }) +client.capabilities = { flat_subdomains: true } describe('doc normalization', () => { it('keeps the highest between rev and _rev and removes the rev attribute', () => { @@ -56,6 +57,36 @@ describe('doc normalization', () => { } }) }) + + it('should normalize apps links', () => { + const normalized = normalizeDoc( + { + _id: 1234, + _rev: '3-deadbeef', + rev: '4-cffee', + slug: 'contact', + version: '1.2.0' + }, + 'io.cozy.apps', + client + ) + expect(normalized).toEqual({ + _id: 1234, + id: 1234, + _rev: '4-cffee', + _type: 'io.cozy.apps', + slug: 'contact', + version: '1.2.0', + relationships: { + referenced_by: undefined + }, + links: { + icon: '/apps/contact/icon/1.2.0', + related: 'https://claude-contact.mycozy.cloud/#/', + self: '/apps/contact' + } + }) + }) }) describe('jsonapi', () => { @@ -140,4 +171,404 @@ describe('jsonapi', () => { expect(lastNormalized.next).toBe(false) }) }) + + describe('normalization', () => { + it('Should normalize single doc', () => { + const res = singleDocRes + const normalized = fromPouchResult({ + res, + withRows: false, + doctype: 'io.cozy.settings', + client + }) + expect(normalized).toEqual({ + data: { + _id: 'io.cozy.settings.flags', + _rev: '1-078d414431314ea48ad6556cad579996', + _type: 'io.cozy.settings', + cozyLocalOnly: true, + id: 'io.cozy.settings.flags', + links: { + self: '/settings/flags' + }, + relationships: { + referenced_by: undefined + }, + 'some.boolean.flag': true, + 'some.number.flag': 30, + 'some.object.flag': { + value1: 100, + value2: 100 + }, + 'some.other.boolean.flag': true, + type: 'io.cozy.settings' + } + }) + }) + }) + it('Should normalize docs array', () => { + const res = multipleDocRes + const normalized = fromPouchResult({ + res, + withRows: true, + doctype: 'io.cozy.files', + client + }) + expect(normalized).toEqual({ + data: [ + { + relationships: { + referenced_by: undefined + }, + id: '018bdcec-00c8-7155-b352-c8a8f472f882', + type: 'file', + _type: 'io.cozy.files', + name: 'New note 2023-11-17T10-55-36Z.cozy-note', + dir_id: '3ab984a52b49806a2a29a14d31cc063f', + created_at: '2023-11-17T10:55:36.061274688Z', + updated_at: '2023-11-17T10:58:40.254562842Z', + size: '51', + md5sum: 'AYU8xZStzHKpiabOg2EHyg==', + mime: 'text/vnd.cozy.note+markdown', + class: 'text', + executable: false, + trashed: false, + encrypted: false, + metadata: { + content: { + content: [], + type: 'doc' + }, + schema: { + marks: [], + nodes: [], + version: 4 + }, + title: '', + version: 57 + }, + cozyMetadata: { + doctypeVersion: '1', + metadataVersion: 1, + createdAt: '2023-11-17T10:55:36.061274688Z', + createdByApp: 'notes', + updatedAt: '2023-11-17T10:58:40.254562842Z', + createdOn: 'https://yannickchironcozywtf1.cozy.wtf/' + }, + internal_vfs_id: 'HyepeHXMIHkKhrmq', + _id: '018bdcec-00c8-7155-b352-c8a8f472f882', + _rev: '2-c78707eb06cceaa5e95c1c4a4c4073bd' + }, + { + relationships: { + referenced_by: [ + { + id: '536bde9aef87dde16630d3c99d26453f', + type: 'io.cozy.photos.albums' + } + ] + }, + id: '018c7cf1-1d00-73ac-9a7f-ee3190638183', + type: 'file', + _type: 'io.cozy.files', + name: 'IMG_0046.jpg', + dir_id: 'io.cozy.files.root-dir', + created_at: '2023-11-19T13:31:47+01:00', + updated_at: '2023-12-18T12:40:25.379Z', + size: '1732841', + md5sum: 'i19eI81lfj3dTwc7i8ihfA==', + mime: 'image/jpeg', + class: 'image', + executable: false, + trashed: false, + encrypted: false, + tags: ['library'], + metadata: { + datetime: '2023-11-19T13:31:47+01:00', + extractor_version: 2, + flash: 'On, Fired', + height: 3024, + orientation: 6, + width: 4032 + }, + referenced_by: [ + { + id: '536bde9aef87dde16630d3c99d26453f', + type: 'io.cozy.photos.albums' + } + ], + cozyMetadata: { + doctypeVersion: '1', + metadataVersion: 1, + createdAt: '2023-12-18T12:40:25.556898681Z', + updatedAt: '2023-12-18T12:45:29.375968305Z', + updatedByApps: [ + { + slug: 'photos', + date: '2023-12-18T12:45:29.375968305Z', + instance: 'https://yannickchironcozywtf1.cozy.wtf/' + } + ], + createdOn: 'https://yannickchironcozywtf1.cozy.wtf/', + uploadedAt: '2023-12-18T12:40:25.556898681Z', + uploadedOn: 'https://yannickchironcozywtf1.cozy.wtf/' + }, + internal_vfs_id: 'SidToiYjmikHrFBP', + _id: '018c7cf1-1d00-73ac-9a7f-ee3190638183', + _rev: '2-02e800df012ea1cc740e5ad1554cefe6' + }, + { + relationships: { + referenced_by: [ + { + id: '536bde9aef87dde16630d3c99d26453f', + type: 'io.cozy.photos.albums' + } + ] + }, + id: '018ca6a8-8292-7acb-bcaf-a95ccfd83662', + _type: 'io.cozy.files', + type: 'file', + name: 'IMG_0047.jpg', + dir_id: 'io.cozy.files.root-dir', + created_at: '2023-11-19T13:31:47+01:00', + updated_at: '2023-12-26T15:05:10.256Z', + size: '1732841', + md5sum: 'i19eI81lfj3dTwc7i8ihfA==', + mime: 'image/jpeg', + class: 'image', + executable: false, + trashed: false, + encrypted: false, + tags: ['library'], + metadata: { + datetime: '2023-11-19T13:31:47+01:00', + extractor_version: 2, + flash: 'On, Fired', + height: 3024, + orientation: 6, + width: 4032 + }, + referenced_by: [ + { + id: '536bde9aef87dde16630d3c99d26453f', + type: 'io.cozy.photos.albums' + } + ], + cozyMetadata: { + doctypeVersion: '1', + metadataVersion: 1, + createdAt: '2023-12-26T15:05:10.51011657Z', + updatedAt: '2025-01-12T12:33:00.313230696Z', + updatedByApps: [ + { + slug: 'photos', + date: '2023-12-26T15:11:03.400641304Z', + instance: 'https://yannickchironcozywtf1.cozy.wtf/' + }, + { + slug: 'drive', + date: '2025-01-12T12:33:00.313230696Z', + instance: 'https://yannickchironcozywtf1.cozy.wtf/' + } + ], + createdOn: 'https://yannickchironcozywtf1.cozy.wtf/', + uploadedAt: '2023-12-26T15:05:10.51011657Z', + uploadedOn: 'https://yannickchironcozywtf1.cozy.wtf/' + }, + internal_vfs_id: 'VSHXbbIKcufNgifx', + _id: '018ca6a8-8292-7acb-bcaf-a95ccfd83662', + _rev: '3-e18fb4f579ba93d569cabcbacc7bcd60' + } + ], + meta: { count: 3 }, + skip: 0, + next: false + }) + }) }) + +const singleDocRes = { + id: 'io.cozy.settings.flags', + type: 'io.cozy.settings', + links: { + self: '/settings/flags' + }, + 'some.boolean.flag': true, + 'some.other.boolean.flag': true, + 'some.object.flag': { + value1: 100, + value2: 100 + }, + 'some.number.flag': 30, + cozyLocalOnly: true, + _id: 'io.cozy.settings.flags', + _rev: '1-078d414431314ea48ad6556cad579996' +} + +const multipleDocRes = { + total_rows: 3, + offset: 0, + rows: [ + { + id: '018bdcec-00c8-7155-b352-c8a8f472f882', + key: '018bdcec-00c8-7155-b352-c8a8f472f882', + value: { + rev: '2-c78707eb06cceaa5e95c1c4a4c4073bd' + }, + doc: { + type: 'file', + name: 'New note 2023-11-17T10-55-36Z.cozy-note', + dir_id: '3ab984a52b49806a2a29a14d31cc063f', + created_at: '2023-11-17T10:55:36.061274688Z', + updated_at: '2023-11-17T10:58:40.254562842Z', + size: '51', + md5sum: 'AYU8xZStzHKpiabOg2EHyg==', + mime: 'text/vnd.cozy.note+markdown', + class: 'text', + executable: false, + trashed: false, + encrypted: false, + metadata: { + content: { + content: [], + type: 'doc' + }, + schema: { + marks: [], + nodes: [], + version: 4 + }, + title: '', + version: 57 + }, + cozyMetadata: { + doctypeVersion: '1', + metadataVersion: 1, + createdAt: '2023-11-17T10:55:36.061274688Z', + createdByApp: 'notes', + updatedAt: '2023-11-17T10:58:40.254562842Z', + createdOn: 'https://yannickchironcozywtf1.cozy.wtf/' + }, + internal_vfs_id: 'HyepeHXMIHkKhrmq', + _id: '018bdcec-00c8-7155-b352-c8a8f472f882', + _rev: '2-c78707eb06cceaa5e95c1c4a4c4073bd' + } + }, + { + id: '018c7cf1-1d00-73ac-9a7f-ee3190638183', + key: '018c7cf1-1d00-73ac-9a7f-ee3190638183', + value: { + rev: '2-02e800df012ea1cc740e5ad1554cefe6' + }, + doc: { + type: 'file', + name: 'IMG_0046.jpg', + dir_id: 'io.cozy.files.root-dir', + created_at: '2023-11-19T13:31:47+01:00', + updated_at: '2023-12-18T12:40:25.379Z', + size: '1732841', + md5sum: 'i19eI81lfj3dTwc7i8ihfA==', + mime: 'image/jpeg', + class: 'image', + executable: false, + trashed: false, + encrypted: false, + tags: ['library'], + metadata: { + datetime: '2023-11-19T13:31:47+01:00', + extractor_version: 2, + flash: 'On, Fired', + height: 3024, + orientation: 6, + width: 4032 + }, + referenced_by: [ + { + id: '536bde9aef87dde16630d3c99d26453f', + type: 'io.cozy.photos.albums' + } + ], + cozyMetadata: { + doctypeVersion: '1', + metadataVersion: 1, + createdAt: '2023-12-18T12:40:25.556898681Z', + updatedAt: '2023-12-18T12:45:29.375968305Z', + updatedByApps: [ + { + slug: 'photos', + date: '2023-12-18T12:45:29.375968305Z', + instance: 'https://yannickchironcozywtf1.cozy.wtf/' + } + ], + createdOn: 'https://yannickchironcozywtf1.cozy.wtf/', + uploadedAt: '2023-12-18T12:40:25.556898681Z', + uploadedOn: 'https://yannickchironcozywtf1.cozy.wtf/' + }, + internal_vfs_id: 'SidToiYjmikHrFBP', + _id: '018c7cf1-1d00-73ac-9a7f-ee3190638183', + _rev: '2-02e800df012ea1cc740e5ad1554cefe6' + } + }, + { + id: '018ca6a8-8292-7acb-bcaf-a95ccfd83662', + key: '018ca6a8-8292-7acb-bcaf-a95ccfd83662', + value: { + rev: '3-e18fb4f579ba93d569cabcbacc7bcd60' + }, + doc: { + type: 'file', + name: 'IMG_0047.jpg', + dir_id: 'io.cozy.files.root-dir', + created_at: '2023-11-19T13:31:47+01:00', + updated_at: '2023-12-26T15:05:10.256Z', + size: '1732841', + md5sum: 'i19eI81lfj3dTwc7i8ihfA==', + mime: 'image/jpeg', + class: 'image', + executable: false, + trashed: false, + encrypted: false, + tags: ['library'], + metadata: { + datetime: '2023-11-19T13:31:47+01:00', + extractor_version: 2, + flash: 'On, Fired', + height: 3024, + orientation: 6, + width: 4032 + }, + referenced_by: [ + { + id: '536bde9aef87dde16630d3c99d26453f', + type: 'io.cozy.photos.albums' + } + ], + cozyMetadata: { + doctypeVersion: '1', + metadataVersion: 1, + createdAt: '2023-12-26T15:05:10.51011657Z', + updatedAt: '2025-01-12T12:33:00.313230696Z', + updatedByApps: [ + { + slug: 'photos', + date: '2023-12-26T15:11:03.400641304Z', + instance: 'https://yannickchironcozywtf1.cozy.wtf/' + }, + { + slug: 'drive', + date: '2025-01-12T12:33:00.313230696Z', + instance: 'https://yannickchironcozywtf1.cozy.wtf/' + } + ], + createdOn: 'https://yannickchironcozywtf1.cozy.wtf/', + uploadedAt: '2023-12-26T15:05:10.51011657Z', + uploadedOn: 'https://yannickchironcozywtf1.cozy.wtf/' + }, + internal_vfs_id: 'VSHXbbIKcufNgifx', + id: '018ca6a8-8292-7acb-bcaf-a95ccfd83662', + _rev: '3-e18fb4f579ba93d569cabcbacc7bcd60' + } + } + ] +} From 83c3d92a5eaef34ab47aa7c53566b5fa773b4cd9 Mon Sep 17 00:00:00 2001 From: Ldoppea Date: Fri, 28 Feb 2025 20:07:28 +0100 Subject: [PATCH 6/8] feat: Edit docs instead of creating a new ones in fromPouchResult --- packages/cozy-pouch-link/src/CozyPouchLink.js | 1 + packages/cozy-pouch-link/src/jsonapi.js | 60 +++++++++++-------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/packages/cozy-pouch-link/src/CozyPouchLink.js b/packages/cozy-pouch-link/src/CozyPouchLink.js index e4594cd2ca..09193c2826 100644 --- a/packages/cozy-pouch-link/src/CozyPouchLink.js +++ b/packages/cozy-pouch-link/src/CozyPouchLink.js @@ -279,6 +279,7 @@ class PouchLink extends CozyLink { * Emits an event (pouchlink:sync:end) when the sync (all doctypes) is done */ handleOnSync(doctypeUpdates) { + // FIXME const normalizedData = mapValues(doctypeUpdates, normalizeAll(this.client)) if (this.client) { this.client.setData(normalizedData) diff --git a/packages/cozy-pouch-link/src/jsonapi.js b/packages/cozy-pouch-link/src/jsonapi.js index a54223ed94..5b0bdab6dd 100644 --- a/packages/cozy-pouch-link/src/jsonapi.js +++ b/packages/cozy-pouch-link/src/jsonapi.js @@ -1,32 +1,40 @@ import { generateWebLink } from 'cozy-client' +const normalizeDocs = (docs, doctype, client) => { + for (let i = docs.length; i >= 0; i--) { + const doc = docs[i] + if (!doc) { + docs.splice(i, 1) + continue + } + normalizeDoc(doc, doctype, client) + } +} + export const normalizeDoc = (doc, doctype, client) => { const id = doc._id || doc.id - - const { relationships, referenced_by } = doc - - // PouchDB sends back .rev attribute but we do not want to - // keep it on the server. It is potentially higher than the - // _rev. const _rev = doc.rev || doc._rev - const normalizedDoc = { - ...doc, - id, - _id: id, - _rev, - _type: doctype, - relationships: { - ...relationships, - referenced_by + doc.id = id + doc._id = id + doc._rev = _rev + doc._type = doctype + + if (doc.relationships) { + doc.relationships.referenced_by = doc.referenced_by + } else { + doc.relationships = { + referenced_by: doc.referenced_by } } - if (normalizedDoc.rev) { - delete normalizedDoc.rev + if (doc.rev) { + delete doc.rev } - normalizeAppsLinks(normalizedDoc, doctype, client) + if (doctype === 'io.cozy.apps') { + normalizeAppsLinks(doc, doctype, client) + } - return normalizedDoc + return doc } const normalizeAppsLinks = (docRef, doctype, client) => { @@ -50,8 +58,6 @@ const normalizeAppsLinks = (docRef, doctype, client) => { } } -const filterDeletedDocumentsFromRows = doc => !!doc - export const fromPouchResult = ({ res, withRows, doctype, client }) => { // Sometimes, queries are transformed by Collections and they call a dedicated // cozy-stack route. When this is the case, we want to be able to replicate the same @@ -68,21 +74,23 @@ export const fromPouchResult = ({ res, withRows, doctype, client }) => { } if (withRows) { - const docs = res.rows - ? res.rows.map(row => row.doc).filter(filterDeletedDocumentsFromRows) - : res.docs + const docs = res.rows ? res.rows.map(row => row.doc) : res.docs const offset = res.offset || 0 + normalizeDocs(docs, doctype) - return { - data: docs.map(doc => normalizeDoc(doc, doctype, client)), + const result = { + data: docs, meta: { count: docs.length }, skip: offset, next: offset + docs.length < res.total_rows || docs.length >= res.limit } + return result } else { return { data: Array.isArray(res) + // FIXME ? res.map(doc => normalizeDoc(doc, doctype, client)) + // FIXME : normalizeDoc(res, doctype, client) } } From 19e969136be476c61ed4a271ce62587ff31d7afa Mon Sep 17 00:00:00 2001 From: Ldoppea Date: Fri, 28 Feb 2025 20:08:46 +0100 Subject: [PATCH 7/8] feat: TEMP use fast merge --- packages/cozy-client/package.json | 1 + packages/cozy-client/src/store/documents.js | 45 ++++++++++++++++++++- yarn.lock | 5 +++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/packages/cozy-client/package.json b/packages/cozy-client/package.json index 6438dead5c..4b2e34162d 100644 --- a/packages/cozy-client/package.json +++ b/packages/cozy-client/package.json @@ -15,6 +15,7 @@ }, "dependencies": { "@cozy/minilog": "1.0.0", + "@fastify/deepmerge": "2.0.2", "@types/jest": "^26.0.20", "@types/lodash": "^4.14.170", "btoa": "^1.2.1", diff --git a/packages/cozy-client/src/store/documents.js b/packages/cozy-client/src/store/documents.js index 8992b0d998..2a89f7d6f5 100644 --- a/packages/cozy-client/src/store/documents.js +++ b/packages/cozy-client/src/store/documents.js @@ -3,6 +3,7 @@ import get from 'lodash/get' import isEqual from 'lodash/isEqual' import omit from 'lodash/omit' import merge from 'lodash/merge' +import deepmerge from '@fastify/deepmerge' import logger from '../logger' @@ -163,24 +164,64 @@ export const getCollectionFromSlice = (state = {}, doctype) => { will be as full of information as it can be. */ export const extractAndMergeDocument = (data, updatedStateWithIncluded) => { + // console.log('πŸͺ extractAndMergeDocument DeepMerge') + const begin = performance.now() const doctype = data[0]._type if (!doctype) { logger.info('Document without _type', data[0]) throw new Error('Document without _type') } + // const beginKeyBy = performance.now() const sortedData = keyBy(data, properId) - + // const endKeyBy = performance.now() + // console.log('πŸͺ extractAndMergeDocument KeyBy took', (endKeyBy - beginKeyBy), 'ms') + const deeppmerge = deepmerge() + // const beginMergeData = performance.now() let mergedData = Object.assign({}, updatedStateWithIncluded) mergedData[doctype] = Object.assign({}, updatedStateWithIncluded[doctype]) + + // console.log('🌈🌈🌈 sortedData: ', sortedData) + // console.log('🌈🌈🌈 mergedData: ', mergedData) + + /* + mergedData[doctype] = deeppmerge(mergedData[doctype], sortedData) + /*/ + // const endMergeData = performance.now() + // console.log('πŸͺ extractAndMergeDocument MergeData took', (endMergeData - beginMergeData), 'ms') + // const beginMapSorted = performance.now() + // let lodashMergeCount = 0 + // let lodashMergeDuration = 0 Object.values(sortedData).map(data => { + // const beginProperId = performance.now() const id = properId(data) + // const endProperId = performance.now() + // console.log('πŸͺ extractAndMergeDocument ProperId took', (endProperId - beginProperId), 'ms') if (mergedData[doctype][id]) { - mergedData[doctype][id] = merge({}, mergedData[doctype][id], data) + if (JSON.stringify(data) !== JSON.stringify(mergedData[doctype][id])) { + // const beginLodashMerge = performance.now() + // mergedData[doctype][id] = merge({}, mergedData[doctype][id], data) + const temp = Object.assign({}, mergedData[doctype][id]) + mergedData[doctype][id] = deeppmerge(temp, data) + // console.log('mergedData[doctype][id]', mergedData[doctype][id]) + // const endLodashMerge = performance.now() + // lodashMergeCount += 1 + // lodashMergeDuration += (endLodashMerge - beginLodashMerge) + } + // console.log('πŸͺ extractAndMergeDocument LodashMerge took', (endLodashMerge - beginLodashMerge), 'ms') } else { + // const beginRawData = performance.now() mergedData[doctype][id] = data + // const endRawData = performance.now() + // console.log('πŸͺ extractAndMergeDocument RawData took', (endRawData - beginRawData), 'ms') } }) + //*/ + // console.log(`πŸͺ🌈 lodashMerge called ${lodashMergeCount} times for ${lodashMergeDuration}ms`) + // const endMapSorted = performance.now() + // console.log('πŸͺ extractAndMergeDocument MapSorted took', (endMapSorted - beginMapSorted), 'ms') + const end = performance.now() + console.log('πŸͺ extractAndMergeDocument took', (end - begin), 'ms') return mergedData } diff --git a/yarn.lock b/yarn.lock index 36cf1a7ebf..7a73709b3e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1235,6 +1235,11 @@ minimatch "^3.0.4" strip-json-comments "^3.1.1" +"@fastify/deepmerge@2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@fastify/deepmerge/-/deepmerge-2.0.2.tgz#5dcbda2acb266e309b8a1ca92fa48b2125e65fc0" + integrity sha512-3wuLdX5iiiYeZWP6bQrjqhrcvBIf0NHbQH1Ur1WbHvoiuTYUEItgygea3zs8aHpiitn0lOB8gX20u1qO+FDm7Q== + "@humanwhocodes/config-array@^0.9.2": version "0.9.5" resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.9.5.tgz#2cbaf9a89460da24b5ca6531b8bbfc23e1df50c7" From 5807df1a7b19a6d918194bc81d0d9c2edb17464a Mon Sep 17 00:00:00 2001 From: Ldoppea Date: Fri, 28 Feb 2025 20:09:32 +0100 Subject: [PATCH 8/8] feat: TEMP ADD PERF LOGS --- packages/cozy-client/src/CozyClient.js | 59 +++++++++++++++---- packages/cozy-client/src/StackLink.js | 3 + packages/cozy-client/src/WebFlagshipLink.js | 1 + packages/cozy-client/src/hooks/utils.js | 1 + packages/cozy-client/src/store/documents.js | 19 +++++- packages/cozy-client/src/store/queries.js | 10 ++++ packages/cozy-pouch-link/src/CozyPouchLink.js | 29 ++++++++- packages/cozy-pouch-link/src/helpers.js | 4 +- .../cozy-stack-client/src/CozyStackClient.js | 13 +++- .../src/SettingsCollection.js | 7 ++- 10 files changed, 130 insertions(+), 16 deletions(-) diff --git a/packages/cozy-client/src/CozyClient.js b/packages/cozy-client/src/CozyClient.js index bfc481d00a..51f2ef9477 100644 --- a/packages/cozy-client/src/CozyClient.js +++ b/packages/cozy-client/src/CozyClient.js @@ -474,6 +474,7 @@ class CozyClient { } async _login(options) { + console.log('πŸŸ₯ login 1') this.emit('beforeLogin') this.registerClientOnLinks() @@ -487,11 +488,13 @@ class CozyClient { } } + console.log('πŸŸ₯ login 2') for (const link of this.links) { if (link.onLogin) { await link.onLogin() } } + console.log('πŸŸ₯ login 3') this.isLogged = true this.isRevoked = false @@ -499,6 +502,7 @@ class CozyClient { if (this.stackClient instanceof OAuthClient) { await this.loadInstanceOptionsFromStack() } + console.log('πŸŸ₯ login 4') this.emit('login') } @@ -926,6 +930,8 @@ client.query(Q('io.cozy.bills'))`) * @returns {Promise} */ async query(queryDefinition, { update, executeFromStore, ...options } = {}) { + console.log('⏰query 1') + const beginPrepare = performance.now() const markQuery = this.performanceApi.mark( `client.query(${queryDefinition.doctype})` ) @@ -976,21 +982,28 @@ client.query(Q('io.cozy.bills'))`) executeQueryFromState(this.store.getState(), queryDefinition) ) : () => this.requestQuery(queryDefinition, options) + const endPrepare = performance.now() + console.log('🍺 CozyClient query prepare took', (endPrepare - beginPrepare), 'ms') + const beginExec = performance.now() const response = await this._promiseCache.exec(requestFn, () => stringify(queryDefinition) ) + const endExec = performance.now() + console.log('🍺 CozyClient query exec took', (endExec - beginExec), 'ms') + const beginReceive = performance.now() + const queryReceivedResult = receiveQueryResult(queryId, response, { + update, + backgroundFetching + }) + const endReceive = performance.now() + console.log('🍺 CozyClient query receiveQueryResult took', (endReceive - beginReceive), 'ms') + const beginDispatch = performance.now() this.dispatch( - receiveQueryResult(queryId, response, { - update, - backgroundFetching - }) + queryReceivedResult ) - this.performanceApi.measure({ - markName: markQuery, - measureName: `${markQuery} success`, - color: 'primary' - }) + const endDispatch = performance.now() + console.log('🍺 CozyClient query Dispatch took', (endDispatch - beginDispatch), 'ms') return response } catch (error) { this.performanceApi.measure({ @@ -1025,10 +1038,14 @@ client.query(Q('io.cozy.bills'))`) options.as || this.queryIdGenerator.generateId(queryDefinition) const mergedOptions = { ...options, as: queryId } try { + const begin = performance.now() let resp = await this.query(queryDefinition, mergedOptions) + const end = performance.now() + console.log('🍺 first query took', (end - begin), 'ms') const documents = resp.data while (resp && resp.next) { + console.log('🍺 while') if (resp.bookmark) { resp = await this.query( queryDefinition.offsetBookmark(resp.bookmark), @@ -1046,6 +1063,7 @@ client.query(Q('io.cozy.bills'))`) } documents.push(...resp.data) } + console.log('🍺 return docs') return documents } catch (e) { logger.log(`queryAll error for ${e.toString()}`) @@ -1111,17 +1129,27 @@ client.query(Q('io.cozy.bills'))`) * @returns {Promise} */ async requestQuery(definition, options) { + const begin = performance.now() const mainResponse = await this.chain.request(definition, options) + const end = performance.now() + console.log('πŸ›‹οΈ CozyClient chain.request took', (end - begin), 'ms') + const begin2 = performance.now() await this.persistVirtualDocuments(definition, mainResponse.data) + const end2 = performance.now() + console.log('πŸ›‹οΈ CozyClient persistVirtualDocuments took', (end2 - begin2), 'ms') if (!definition.includes) { + console.log('πŸ›‹οΈ CozyClient return no-includes') return mainResponse } + const begin3 = performance.now() const withIncluded = await this.fetchRelationships( mainResponse, this.getIncludesRelationships(definition) ) + const end3 = performance.now() + console.log('πŸ›‹οΈ CozyClient fetchRelationships took', (end3 - begin3), 'ms') return withIncluded } @@ -1188,6 +1216,7 @@ client.query(Q('io.cozy.bills'))`) } if ((!document.meta?.rev && !document._rev) || enforce) { + console.log('🚨🚨🚨🚨 PERSIST') await this.chain.persistCozyData(document) } } @@ -1814,12 +1843,20 @@ instantiation of the client.` * @returns {Promise} */ async loadInstanceOptionsFromStack() { + console.log('πŸ”° loadInstanceOptionsFromStack 1') const results = await Promise.all([ this.query( Q('io.cozy.settings').getById('io.cozy.settings.capabilities') - ), - this.query(Q('io.cozy.settings').getById('io.cozy.settings.instance')) + ).catch(error => { + console.log(error) + throw error + }), + this.query(Q('io.cozy.settings').getById('io.cozy.settings.instance')).catch(error => { + console.log(error) + throw error + }) ]) + console.log('πŸ”° loadInstanceOptionsFromStack 2') const { data: capabilitiesData } = results[0] const { data: instanceData } = results[1] diff --git a/packages/cozy-client/src/StackLink.js b/packages/cozy-client/src/StackLink.js index 079cb270b3..aaf20ccc29 100644 --- a/packages/cozy-client/src/StackLink.js +++ b/packages/cozy-client/src/StackLink.js @@ -90,7 +90,9 @@ export default class StackLink extends CozyLink { } async request(operation, options, result, forward) { + console.log('stack request') if (!options?.forceStack && this.isOnline && !(await this.isOnline())) { + console.log('forward') return forward(operation, options) } @@ -108,6 +110,7 @@ export default class StackLink extends CozyLink { } async persistCozyData(data, forward) { + console.log('persistCozyData from StackLink') return forward(data) } /** diff --git a/packages/cozy-client/src/WebFlagshipLink.js b/packages/cozy-client/src/WebFlagshipLink.js index 765566195c..e4d30e7a01 100644 --- a/packages/cozy-client/src/WebFlagshipLink.js +++ b/packages/cozy-client/src/WebFlagshipLink.js @@ -23,6 +23,7 @@ export default class WebFlagshipLink extends CozyLink { } async persistCozyData(data, forward) { + console.log('persistCozyData from WebFlagshipLink') // Persist data should do nothing here as data is already persisted on Flagship side return } diff --git a/packages/cozy-client/src/hooks/utils.js b/packages/cozy-client/src/hooks/utils.js index 8904f6997f..91e3e5fdf2 100644 --- a/packages/cozy-client/src/hooks/utils.js +++ b/packages/cozy-client/src/hooks/utils.js @@ -10,6 +10,7 @@ * @returns */ export const equalityCheckForQuery = (queryResA, queryResB) => { + // console.log('🌈 Equality check for query') //console.log('Call equality check : ', queryResA, queryResB) if (queryResA === queryResB) { // Referential equality diff --git a/packages/cozy-client/src/store/documents.js b/packages/cozy-client/src/store/documents.js index 2a89f7d6f5..de01fe151f 100644 --- a/packages/cozy-client/src/store/documents.js +++ b/packages/cozy-client/src/store/documents.js @@ -14,6 +14,7 @@ import { isReceivingMutationResult } from './mutations' import { properId } from './helpers' const storeDocument = (state, document) => { + console.log('πŸͺ storeDocument') const type = document._type if (!type) { if (process.env.NODE_ENV !== 'production') { @@ -50,6 +51,7 @@ export const mergeDocumentsWithRelationships = ( prevDocument = {}, nextDocument = {} ) => { + console.log('πŸͺ mergeDocumentsWithRelationships') /** * @type {import("../types").CozyClientDocument} */ @@ -69,6 +71,8 @@ export const mergeDocumentsWithRelationships = ( // reducer const documents = (state = {}, action) => { + // console.log('πŸͺ documents reducer') + // const begin = performance.now() if (!isReceivingData(action) && !isReceivingMutationResult(action)) { return state } @@ -80,6 +84,9 @@ const documents = (state = {}, action) => { ) { const docId = action.definition.document._id const _type = action.definition.document._type + + // const end = performance.now() + // console.log('πŸͺ documents reducer 1 took', (end - begin), 'ms') return { ...state, [_type]: omit(state[_type], docId) @@ -87,14 +94,22 @@ const documents = (state = {}, action) => { } const { data, included } = action.response - if (!data || (Array.isArray(data) && data.length === 0)) return state + if (!data || (Array.isArray(data) && data.length === 0)) { + // const end = performance.now() + // console.log('πŸͺ documents reducer 2 took', (end - begin), 'ms') + return state + } const updatedStateWithIncluded = included ? included.reduce(storeDocument, state) : state if (!Array.isArray(data)) { + // const end = performance.now() + // console.log('πŸͺ documents reducer 3 took', (end - begin), 'ms') return storeDocument(updatedStateWithIncluded, data) } + // const end = performance.now() + // console.log('πŸͺ documents reducer 4 took', (end - begin), 'ms') return extractAndMergeDocument(data, updatedStateWithIncluded) } @@ -102,6 +117,7 @@ export default documents // selector export const getDocumentFromSlice = (state = {}, doctype, id) => { + // console.log('πŸͺ getDocumentFromSlice') if (!doctype) { throw new Error( 'getDocumentFromSlice: Cannot retrieve document with undefined doctype' @@ -133,6 +149,7 @@ export const getDocumentFromSlice = (state = {}, doctype, id) => { } export const getCollectionFromSlice = (state = {}, doctype) => { + // console.log('πŸͺ getCollectionFromSlice') if (!doctype) { throw new Error( 'getDocumentFromSlice: Cannot retrieve document with undefined doctype' diff --git a/packages/cozy-client/src/store/queries.js b/packages/cozy-client/src/store/queries.js index 5b59351299..40eda2a69f 100644 --- a/packages/cozy-client/src/store/queries.js +++ b/packages/cozy-client/src/store/queries.js @@ -140,6 +140,7 @@ const query = ( action, documents ) => { + // console.log('πŸͺ query', action.type) switch (action.type) { case INIT_QUERY: if ( @@ -172,12 +173,15 @@ const query = ( fetchStatus: 'loading' } case RECEIVE_QUERY_RESULT: { + // const begin = performance.now() const markName = performanceApi.mark('RECEIVE_QUERY_RESULT') const response = action.response // Data can be null when we get a 404 not found // see Collection.get() // but we still need to update the fetchStatus. if (!response.data) { + // const end = performance.now() + // console.log('πŸͺ RECEIVE_QUERY_RESULT with no data took', (end - begin), 'ms') performanceApi.measure({ markName: markName, measureName: `${markName} with no data`, @@ -204,6 +208,8 @@ const query = ( } if (!Array.isArray(response.data)) { + // const end = performance.now() + // console.log('πŸͺ RECEIVE_QUERY_RESULT with object took', (end - begin), 'ms') performanceApi.measure({ markName: markName, measureName: `${markName} with object`, @@ -227,6 +233,8 @@ const query = ( measureName: `${markName} with background fetching`, category: 'CozyClientStore' }) + // const end = performance.now() + // console.log('πŸͺ RECEIVE_QUERY_RESULT with background fetching took', (end - begin), 'ms') return { ...state, ...common, @@ -243,6 +251,8 @@ const query = ( fetchedPagesCount }) + // const end = performance.now() + // console.log('πŸͺ RECEIVE_QUERY_RESULT default took', (end - begin), 'ms') performanceApi.measure({ markName: markName, measureName: `${markName} default`, diff --git a/packages/cozy-pouch-link/src/CozyPouchLink.js b/packages/cozy-pouch-link/src/CozyPouchLink.js index 09193c2826..bedcc574aa 100644 --- a/packages/cozy-pouch-link/src/CozyPouchLink.js +++ b/packages/cozy-pouch-link/src/CozyPouchLink.js @@ -279,6 +279,7 @@ class PouchLink extends CozyLink { * Emits an event (pouchlink:sync:end) when the sync (all doctypes) is done */ handleOnSync(doctypeUpdates) { + console.log('onSync', doctypeUpdates) // FIXME const normalizedData = mapValues(doctypeUpdates, normalizeAll(this.client)) if (this.client) { @@ -307,6 +308,7 @@ class PouchLink extends CozyLink { * @private */ _startReplication({ waitForReplications = true } = {}) { + console.log('StartReplication') this.client.emit('pouchlink:sync:start') if (this.periodicSync) { // FIXME: this API is kind of weird, one should be able to manually replicate @@ -420,6 +422,7 @@ class PouchLink extends CozyLink { } async request(operation, options, result = null, forward = doNothing) { + const begin = performance.now() if (options?.forceStack) { return forward(operation, options) } @@ -464,10 +467,17 @@ class PouchLink extends CozyLink { return forward(operation, options) } + const end = performance.now() + console.log('🍺 Pouch.request preparation took', (end - begin), 'ms') if (operation.mutationType) { return this.executeMutation(operation, options, result, forward) } else { - return this.executeQuery(operation) + + const beginR = performance.now() + const result = await this.executeQuery(operation) + const endR = performance.now() + console.log('🍺 Pouch.request executeQuery took', (endR - beginR), 'ms') + return result } } @@ -530,6 +540,7 @@ class PouchLink extends CozyLink { } async persistCozyData(data, forward = doNothing) { + console.log('persistCozyData from CozyPouchLink') const markName = this.performanceApi.mark('persistCozyData') const sanitizedDoc = this.sanitizeJsonApi(data) sanitizedDoc.cozyLocalOnly = true @@ -699,7 +710,10 @@ class PouchLink extends CozyLink { partialFilter }) { const markName = this.performanceApi.mark('executeQuery') + const beginPrepare = performance.now() const db = this.getPouch(doctype) + const endPrepare = performance.now() + console.log('πŸ›‹οΈ executeQuery getPouch took', (endPrepare - beginPrepare), 'ms') let res, withRows if (id) { const markName = this.performanceApi.mark('db.get from executeQuery') @@ -710,6 +724,7 @@ class PouchLink extends CozyLink { const markName = this.performanceApi.mark( 'allDocs from executeQuery with ids' ) + console.log('πŸ›‹οΈ allDocs1') res = await allDocs(db, { include_docs: true, keys: ids }) this.performanceApi.measure({ markName, category: 'PouchDB' }) res = withoutDesignDocuments(res) @@ -717,10 +732,17 @@ class PouchLink extends CozyLink { withRows = true } else if (!selector && !partialFilter && !fields && !sort) { const markName = this.performanceApi.mark('allDocs from executeQuery') + const begin = performance.now() res = await allDocs(db, { include_docs: true, limit }) + const end = performance.now() + console.log('πŸ›‹οΈ executeQuery allDocs2 took', (end - begin), 'ms (include db[fct]') this.performanceApi.measure({ markName, category: 'PouchDB' }) + + const begin3 = performance.now() res = withoutDesignDocuments(res) withRows = true + const end3 = performance.now() + console.log('πŸ›‹οΈ executeQuery withoutDesignDocuments took', (end3 - begin3), 'ms') } else { const findSelector = helpers.normalizeFindSelector({ selector, @@ -745,18 +767,23 @@ class PouchLink extends CozyLink { }) findOpts.use_index = index.id const markName = this.performanceApi.mark('find from executeQuery') + console.log('πŸ›‹οΈ find') res = await find(db, findOpts) this.performanceApi.measure({ markName, category: 'PouchDB' }) res.offset = skip res.limit = limit withRows = true } + + const begin2 = performance.now() const jsonResult = jsonapi.fromPouchResult({ res, withRows, doctype, client: this.client }) + const end2 = performance.now() + console.log('πŸ›‹οΈ executeQuery jsonapi.fromPouchResult took', (end2 - begin2), 'ms') this.performanceApi.measure({ markName: markName, diff --git a/packages/cozy-pouch-link/src/helpers.js b/packages/cozy-pouch-link/src/helpers.js index f630efbe05..74764afad9 100644 --- a/packages/cozy-pouch-link/src/helpers.js +++ b/packages/cozy-pouch-link/src/helpers.js @@ -36,8 +36,10 @@ helpers.getDocs = async (db, fct, options = {}) => { options.skip = options.skip || 0 } } - + const begin = performance.now() const data = await db[fct](options) + const end = performance.now() + console.log('πŸ›‹οΈ db[fct](options) took', (end - begin), 'ms') if (data[field].length === options.limit) { options.skip = (options.skip ? options.skip : 0) + options.limit diff --git a/packages/cozy-stack-client/src/CozyStackClient.js b/packages/cozy-stack-client/src/CozyStackClient.js index b37b277219..0c955f4493 100644 --- a/packages/cozy-stack-client/src/CozyStackClient.js +++ b/packages/cozy-stack-client/src/CozyStackClient.js @@ -157,9 +157,13 @@ class CozyStackClient { : fetch try { + console.log('🧑 fetcher1') const response = await fetcher(fullPath, options) + console.log('🧑 fetcher2') if (!response.ok) { + console.log('🧑 fetcher3') const reason = await getResponseData(response) + console.log('🧑 fetcher4') const err = new FetchError(response, reason) // XXX: This was introduced so apps could display errors (e.g. quota @@ -176,8 +180,10 @@ class CozyStackClient { // We could then get rid of `throwFetchErrors`. if (throwFetchErrors) throw err } + console.log('🧑 fetcher5') return response } catch (err) { + console.log('🧑 fetcher err') if (this.isRevocationError(err)) { this.onRevocationChange(true) } @@ -253,7 +259,10 @@ class CozyStackClient { */ async fetchJSON(method, path, body, options = {}) { try { - return await this.fetchJSONWithCurrentToken(method, path, body, options) + console.log('fetchJSON 1', path) + const result = await this.fetchJSONWithCurrentToken(method, path, body, options) + console.log('fetchJSON 2', path) + return result } catch (e) { if ( errors.EXPIRED_TOKEN.test(e.message) || @@ -290,7 +299,9 @@ class CozyStackClient { } } clonedOptions.throwFetchErrors = true + console.log('πŸ”΅ fetchJSONWithCurrentToken1' + path) const resp = await this.fetch(method, path, body, clonedOptions) + console.log('πŸ”΅ fetchJSONWithCurrentToken2') return getResponseData(resp) } diff --git a/packages/cozy-stack-client/src/SettingsCollection.js b/packages/cozy-stack-client/src/SettingsCollection.js index 93735ceb55..c37f13d666 100644 --- a/packages/cozy-stack-client/src/SettingsCollection.js +++ b/packages/cozy-stack-client/src/SettingsCollection.js @@ -54,13 +54,18 @@ class SettingsCollection extends DocumentCollection { path = id } + console.log('πŸ€– fetchJSON get Settings' + path) const resp = await this.stackClient.fetchJSON('GET', `/settings/${path}`) - return { + console.log('πŸ€– fetchJSON get Settings2' + path) + const result = { data: normalizeSettings({ id: `/settings/${path}`, ...resp.data }) } + + console.log('πŸ€– fetchJSON get Settings3' + path) + return result } /**