From 1b996a8faac7c76ee7a067e91896cecea5ac263e Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Fri, 11 Apr 2025 19:16:31 +0200 Subject: [PATCH 1/7] refactor: Move mutualizable code --- packages/cozy-pouch-link/src/db/helpers.js | 25 ++++++++++++++ .../cozy-pouch-link/src/db/helpers.spec.js | 33 +++++++++++++++++++ packages/cozy-pouch-link/src/db/sqlite/sql.js | 27 +-------------- .../cozy-pouch-link/src/db/sqlite/sql.spec.js | 33 ------------------- 4 files changed, 59 insertions(+), 59 deletions(-) create mode 100644 packages/cozy-pouch-link/src/db/helpers.spec.js diff --git a/packages/cozy-pouch-link/src/db/helpers.js b/packages/cozy-pouch-link/src/db/helpers.js index 24730c415c..bfcdbe69eb 100644 --- a/packages/cozy-pouch-link/src/db/helpers.js +++ b/packages/cozy-pouch-link/src/db/helpers.js @@ -55,3 +55,28 @@ export const areDocsEqual = async (oldDoc, newDoc) => { export const getCozyPouchData = doc => { return doc.cozyPouchData } + +const extractRevPrefix = rev => { + if (!rev) { + return 0 + } + const prefixStr = rev.split('-')[0] + return prefixStr ? parseInt(prefixStr) : 0 +} + +export const keepDocWitHighestRev = docs => { + if (!docs || docs.length < 1) { + return null + } + let highestDocRev = { + doc: docs[0], + revPrefix: extractRevPrefix(docs[0]._rev) + } + for (let i = 0; i < docs.length; i++) { + const revPrefix = extractRevPrefix(docs[i]._rev) + if (revPrefix > highestDocRev.revPrefix) { + highestDocRev = { doc: docs[i], revPrefix } + } + } + return highestDocRev.doc +} diff --git a/packages/cozy-pouch-link/src/db/helpers.spec.js b/packages/cozy-pouch-link/src/db/helpers.spec.js new file mode 100644 index 0000000000..fc092b79c4 --- /dev/null +++ b/packages/cozy-pouch-link/src/db/helpers.spec.js @@ -0,0 +1,33 @@ +import { keepDocWitHighestRev } from './helpers' + +describe('keepDocWitHighestRev', () => { + it('should return null if no docs', () => { + expect(keepDocWitHighestRev([])).toBeNull() + expect(keepDocWitHighestRev(undefined)).toBeNull() + }) + + it('should return the single document when only one is provided', () => { + const doc = { _rev: '1-a', name: 'Single Doc' } + const docs = [doc] + expect(keepDocWitHighestRev(docs)).toBe(doc) + }) + + it('should return the document with the highest revision prefix', () => { + const docs = [ + { _rev: '1-a', name: 'Doc 1' }, + { _rev: '3-c', name: 'Doc 3' }, + { _rev: '2-b', name: 'Doc 2' } + ] + expect(keepDocWitHighestRev(docs)).toEqual(docs[1]) + }) + + it('should work correctly even if the documents are unsorted', () => { + const docs = [ + { _rev: '5-zzz', name: 'Doc 5' }, + { _rev: '2-aaa', name: 'Doc 2' }, + { _rev: '10-xxx', name: 'Doc 10' }, + { _rev: '7-bbb', name: 'Doc 7' } + ] + expect(keepDocWitHighestRev(docs)).toEqual(docs[2]) + }) +}) diff --git a/packages/cozy-pouch-link/src/db/sqlite/sql.js b/packages/cozy-pouch-link/src/db/sqlite/sql.js index 9b9348689e..9448c460e5 100644 --- a/packages/cozy-pouch-link/src/db/sqlite/sql.js +++ b/packages/cozy-pouch-link/src/db/sqlite/sql.js @@ -1,5 +1,5 @@ import { normalizeDoc } from '../../jsonapi' -import { getCozyPouchData } from '../helpers' +import { getCozyPouchData, keepDocWitHighestRev } from '../helpers' const MANGO_TO_SQL_OP = { $eq: '=', @@ -13,31 +13,6 @@ const MANGO_TO_SQL_OP = { $exists: 'IS' } -const extractRevPrefix = rev => { - if (!rev) { - return 0 - } - const prefixStr = rev.split('-')[0] - return prefixStr ? parseInt(prefixStr) : 0 -} - -export const keepDocWitHighestRev = docs => { - if (!docs || docs.length < 1) { - return null - } - let highestDocRev = { - doc: docs[0], - revPrefix: extractRevPrefix(docs[0]._rev) - } - for (let i = 0; i < docs.length; i++) { - const revPrefix = extractRevPrefix(docs[i]._rev) - if (revPrefix > highestDocRev.revPrefix) { - highestDocRev = { doc: docs[i], revPrefix } - } - } - return highestDocRev.doc -} - export const parseResults = ( client, result, diff --git a/packages/cozy-pouch-link/src/db/sqlite/sql.spec.js b/packages/cozy-pouch-link/src/db/sqlite/sql.spec.js index ad30394c23..5624ea95bd 100644 --- a/packages/cozy-pouch-link/src/db/sqlite/sql.spec.js +++ b/packages/cozy-pouch-link/src/db/sqlite/sql.spec.js @@ -3,7 +3,6 @@ import { makeWhereClause, makeSortClause, makeSQLQueryFromMango, - keepDocWitHighestRev, makeSQLQueryAll, parseResults } from './sql' @@ -225,38 +224,6 @@ describe('makeSQLQueryAll', () => { }) }) -describe('keepDocWitHighestRev', () => { - it('should return null if no docs', () => { - expect(keepDocWitHighestRev([])).toBeNull() - expect(keepDocWitHighestRev(undefined)).toBeNull() - }) - - it('should return the single document when only one is provided', () => { - const doc = { _rev: '1-a', name: 'Single Doc' } - const docs = [doc] - expect(keepDocWitHighestRev(docs)).toBe(doc) - }) - - it('should return the document with the highest revision prefix', () => { - const docs = [ - { _rev: '1-a', name: 'Doc 1' }, - { _rev: '3-c', name: 'Doc 3' }, - { _rev: '2-b', name: 'Doc 2' } - ] - expect(keepDocWitHighestRev(docs)).toEqual(docs[1]) - }) - - it('should work correctly even if the documents are unsorted', () => { - const docs = [ - { _rev: '5-zzz', name: 'Doc 5' }, - { _rev: '2-aaa', name: 'Doc 2' }, - { _rev: '10-xxx', name: 'Doc 10' }, - { _rev: '7-bbb', name: 'Doc 7' } - ] - expect(keepDocWitHighestRev(docs)).toEqual(docs[2]) - }) -}) - describe('parseResults', () => { const client = {} const doctype = 'testdoctype' From 11771677d478d1815b6a09be2f48143e479032ba Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Fri, 11 Apr 2025 19:16:47 +0200 Subject: [PATCH 2/7] fix: No need for doctype --- packages/cozy-pouch-link/src/db/pouchdb/pouchdb.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cozy-pouch-link/src/db/pouchdb/pouchdb.js b/packages/cozy-pouch-link/src/db/pouchdb/pouchdb.js index cb02fba48c..56650c93f9 100644 --- a/packages/cozy-pouch-link/src/db/pouchdb/pouchdb.js +++ b/packages/cozy-pouch-link/src/db/pouchdb/pouchdb.js @@ -49,7 +49,7 @@ export default class PouchDBQueryEngine extends DatabaseQueryEngine { } async find(options) { - const { selector, sort, partialFilter, doctype } = options + const { selector, sort, partialFilter } = options let { indexedFields } = options indexedFields = getIndexFields({ @@ -68,7 +68,7 @@ export default class PouchDBQueryEngine extends DatabaseQueryEngine { selector, sort, partialFilter, - doctype, + doctype: this.doctype, use_index: indexName, ...options } @@ -87,7 +87,7 @@ export default class PouchDBQueryEngine extends DatabaseQueryEngine { if (isMissingPouchDBIndexError(err)) { await createIndex(this.db, indexedFields, { indexName, - doctype, + doctype: this.doctype, partialFilter }) res = await getDocsAndNormalize({ From d36239e61af4f0fe008613c9a4c102bcd723aac5 Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Thu, 24 Apr 2025 09:52:41 +0200 Subject: [PATCH 3/7] docs: Db methods can return null --- packages/cozy-pouch-link/src/db/dbInterface.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cozy-pouch-link/src/db/dbInterface.js b/packages/cozy-pouch-link/src/db/dbInterface.js index c48f916667..a09afcae42 100644 --- a/packages/cozy-pouch-link/src/db/dbInterface.js +++ b/packages/cozy-pouch-link/src/db/dbInterface.js @@ -50,7 +50,7 @@ class DatabaseQueryEngine { * Get all docs * * @param {AllDocsParams} options - The all docs options - * @returns {Promise} The found docs + * @returns {Promise} The found docs */ async allDocs(options) { throw new Error('method not implemented') @@ -60,7 +60,7 @@ class DatabaseQueryEngine { * Get a single doc by its id * * @param {string} id - id of the document to get - * @returns {Promise} The found docs + * @returns {Promise} The found docs */ async getById(id) { throw new Error('method not implemented') @@ -70,7 +70,7 @@ class DatabaseQueryEngine { * Get several docs by their ids * * @param {Array} ids - ids of the documents to get - * @returns {Promise} The found docs + * @returns {Promise} The found docs */ async getByIds(ids) { throw new Error('method not implemented') From 54b892ecdf37736f4497eb3a824e92a2ddde2779 Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Thu, 24 Apr 2025 11:16:04 +0200 Subject: [PATCH 4/7] feat: Add pouch-find for web platform --- packages/cozy-pouch-link/src/platformWeb.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/cozy-pouch-link/src/platformWeb.js b/packages/cozy-pouch-link/src/platformWeb.js index ef2c85deee..ea35997048 100644 --- a/packages/cozy-pouch-link/src/platformWeb.js +++ b/packages/cozy-pouch-link/src/platformWeb.js @@ -1,6 +1,9 @@ import PouchDB from 'pouchdb-browser' +import PouchDBFind from 'pouchdb-find' import { LOCALSTORAGE_STORAGE_KEYS } from './localStorage' +PouchDB.plugin(PouchDBFind) + const events = { addEventListener: (eventName, handler) => { document.addEventListener(eventName, handler) From c89fe488cf63030afe77ab34c8256f78cd25bed8 Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Thu, 24 Apr 2025 11:12:30 +0200 Subject: [PATCH 5/7] fix: Handle $gt: null for pouchdb PouchDB sometimes badly handle the `$gt: null` selector, thus we transform it into `$gt: ''` See https://github.com/pouchdb/pouchdb/issues/7192 --- packages/cozy-pouch-link/src/helpers.js | 6 +++--- packages/cozy-pouch-link/src/helpers.spec.js | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/cozy-pouch-link/src/helpers.js b/packages/cozy-pouch-link/src/helpers.js index e3e73ae585..505b87674a 100644 --- a/packages/cozy-pouch-link/src/helpers.js +++ b/packages/cozy-pouch-link/src/helpers.js @@ -36,7 +36,7 @@ helpers.normalizeFindSelector = ({ `${indexedField} was missing in selector, it has been automatically added from indexed fields. Please consider adding this field to your query's selector as required by PouchDB. The query's selector is: ${selectorJson}` ) findSelector[indexedField] = { - $gt: null + $gt: '' // See https://github.com/pouchdb/pouchdb/issues/7192 } } } @@ -51,7 +51,7 @@ helpers.normalizeFindSelector = ({ `${sortedField} was missing in selector, it has been automatically added from sorted fields. Please consider adding this field to your query's selector as required by PouchDB. The query's selector is: ${selectorJson}` ) findSelector[sortedField] = { - $gt: null + $gt: '' // See https://github.com/pouchdb/pouchdb/issues/7192 } } } @@ -63,7 +63,7 @@ helpers.normalizeFindSelector = ({ return Object.keys(mergedSelector).length > 0 ? mergedSelector - : { _id: { $gt: null } } // PouchDB does not accept empty selector + : { _id: { $gt: '' } } // PouchDB does not accept empty selector } export default helpers diff --git a/packages/cozy-pouch-link/src/helpers.spec.js b/packages/cozy-pouch-link/src/helpers.spec.js index be855e712c..20f29c6a9d 100644 --- a/packages/cozy-pouch-link/src/helpers.spec.js +++ b/packages/cozy-pouch-link/src/helpers.spec.js @@ -25,7 +25,7 @@ describe('Helpers', () => { describe('normalizeFindSelector', () => { it('should add indexed fields in the selector if they are missing', () => { const selector = { - SOME_FIELD: { $gt: null } + SOME_FIELD: { $gt: '' } } const sort = undefined const indexedFields = ['SOME_INDEXED_FIELD'] @@ -36,8 +36,8 @@ describe('Helpers', () => { indexedFields }) expect(findSelector).toStrictEqual({ - SOME_FIELD: { $gt: null }, - SOME_INDEXED_FIELD: { $gt: null } + SOME_FIELD: { $gt: '' }, + SOME_INDEXED_FIELD: { $gt: '' } }) }) @@ -52,7 +52,7 @@ describe('Helpers', () => { indexedFields }) expect(findSelector).toStrictEqual({ - SOME_SORTED_FIELD: { $gt: null } + SOME_SORTED_FIELD: { $gt: '' } }) }) @@ -67,8 +67,8 @@ describe('Helpers', () => { indexedFields }) expect(findSelector).toStrictEqual({ - SOME_INDEXED_FIELD: { $gt: null }, - SOME_SORTED_FIELD: { $gt: null } + SOME_INDEXED_FIELD: { $gt: '' }, + SOME_SORTED_FIELD: { $gt: '' } }) }) @@ -82,7 +82,7 @@ describe('Helpers', () => { sort, indexedFields }) - expect(findSelector).toStrictEqual({ _id: { $gt: null } }) + expect(findSelector).toStrictEqual({ _id: { $gt: '' } }) }) it('should not add selector on _id when no selector is provided but there are some indexed fields', () => { @@ -96,7 +96,7 @@ describe('Helpers', () => { indexedFields }) expect(findSelector).toStrictEqual({ - SOME_INDEXED_FIELD: { $gt: null } + SOME_INDEXED_FIELD: { $gt: '' } }) }) }) From e7859c1589f76f92523ca48901d100b239f88141 Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Fri, 11 Apr 2025 19:18:16 +0200 Subject: [PATCH 6/7] feat: Add idb backend --- packages/cozy-pouch-link/package.json | 4 +- .../cozy-pouch-link/src/db/idb/getDocs.js | 207 +++++++++ .../src/db/idb/getDocs.spec.js | 414 ++++++++++++++++++ packages/cozy-pouch-link/src/db/idb/idb.js | 167 +++++++ packages/cozy-pouch-link/src/db/idb/mango.js | 368 ++++++++++++++++ .../cozy-pouch-link/src/db/idb/mango.spec.js | 227 ++++++++++ packages/cozy-pouch-link/src/errors.js | 5 + packages/cozy-pouch-link/src/index.js | 1 + yarn.lock | 16 + 9 files changed, 1408 insertions(+), 1 deletion(-) create mode 100644 packages/cozy-pouch-link/src/db/idb/getDocs.js create mode 100644 packages/cozy-pouch-link/src/db/idb/getDocs.spec.js create mode 100644 packages/cozy-pouch-link/src/db/idb/idb.js create mode 100644 packages/cozy-pouch-link/src/db/idb/mango.js create mode 100644 packages/cozy-pouch-link/src/db/idb/mango.spec.js diff --git a/packages/cozy-pouch-link/package.json b/packages/cozy-pouch-link/package.json index 1afb547543..3ee4d61460 100644 --- a/packages/cozy-pouch-link/package.json +++ b/packages/cozy-pouch-link/package.json @@ -15,13 +15,15 @@ "dependencies": { "cozy-client": "^57.7.0", "pouchdb-browser": "^7.2.2", - "pouchdb-find": "^7.2.2" + "pouchdb-find": "^7.2.2", + "sift": "^17.1.3" }, "devDependencies": { "@babel/cli": "7.12.8", "@cozy/minilog": "1.0.0", "@op-engineering/op-sqlite": "^11.4.8", "cozy-device-helper": "2.7.0", + "fake-indexeddb": "^6.0.0", "jest-localstorage-mock": "2.4.19", "parcel": "2.13.3", "pouchdb-adapter-memory": "7.2.2", diff --git a/packages/cozy-pouch-link/src/db/idb/getDocs.js b/packages/cozy-pouch-link/src/db/idb/getDocs.js new file mode 100644 index 0000000000..15a8df2fb7 --- /dev/null +++ b/packages/cozy-pouch-link/src/db/idb/getDocs.js @@ -0,0 +1,207 @@ +import logger from '../../logger' + +import { isMissingIDBIndexError } from '../../errors' +import { executeIDBFind } from './mango' +import { getCozyPouchData, keepDocWitHighestRev } from '../helpers' +import { normalizeDoc } from '../../jsonapi' + +export const MAX_LIMIT = Math.pow(2, 32) - 2 + +export const parseResults = ( + client, + result, + doctype, + { isSingleDoc = false, skip = 0, limit = MAX_LIMIT } = {} +) => { + let parsedResults = [] + for (let i = 0; i < result.length; i++) { + const doc = result[i].data || result[i] + if (!doc._id) { + doc._id = result[i].id + } + if (!doc._rev) { + doc._rev = result[i].rev + } + + // Handle special case for docs with `cozyPouchData` + const cozyPouchData = getCozyPouchData(doc) + if (cozyPouchData) { + return { data: cozyPouchData } + } + + normalizeDoc(client, doctype, doc) + parsedResults.push(doc) + } + if (parsedResults.length === 0) { + return { data: [] } + } + if (isSingleDoc) { + if (parsedResults.length > 1) { + const doc = keepDocWitHighestRev(parsedResults) + return { data: doc } + } + return { data: parsedResults[0] } + } + // XXX - Ideally we should have the total number of rows in the database to have a reliable + // next parameter, but we prefer to avoid this computation for performances. + // So let's rely on the total number of returned rows - if next is true, the last paginated + // query should have less results than the limit, thanks to the offset + let next = false + if (limit !== MAX_LIMIT && parsedResults.length >= limit) { + next = true + } + return { + data: parsedResults, + meta: { count: parsedResults.length }, + skip, + next + } +} + +export const queryWithCursor = async ( + index, + idbKeyRange, + { offset = 0, limit = MAX_LIMIT, sortDirection = 'next' } = {} +) => { + return new Promise((resolve, reject) => { + const result = [] + let advanced = false + //let cursorRequest = index.openCursor() // TODO: 1st param is key range, 2nd param is direction, for sort + const cursorRequest = idbKeyRange + ? index.openCursor(idbKeyRange, sortDirection) + : index.openCursor(null, sortDirection) + + cursorRequest.onsuccess = event => { + const cursor = event.target.result + if (!cursor || result.length >= limit) { + resolve(result) + return + } + + if (!advanced && offset > 0) { + cursor.advance(offset) + advanced = true + } else { + result.push(cursor.value) + cursor.continue() + } + } + + cursorRequest.onerror = event => { + reject(event.target.error) + } + }) +} + +export const queryWithAll = async (store, { limit = MAX_LIMIT } = {}) => { + return new Promise((resolve, reject) => { + const request = store.getAll(null, limit) + + request.onsuccess = event => { + const docs = event.target.result + resolve(docs) + } + + request.onerror = event => { + reject(event.target.error) + } + }) +} + +export const getAllData = async ( + store, + { limit = MAX_LIMIT, skip = 0 } = {} +) => { + let results = [] + const startQ = performance.now() + if (skip === 0) { + results = await queryWithAll(store, { limit }) + } else { + results = await queryWithCursor(store, null, { limit, offset: skip }) + } + const endQ = performance.now() + console.log(`All data took : ${endQ - startQ} ms`) + console.log('length : ', results.length) + + return results +} + +export const findData = async (store, findOpts) => { + const startQ = performance.now() + const results = await executeIDBFind(store, findOpts) + const endQ = performance.now() + console.log(`Find data took : ${endQ - startQ} ms`) + return results +} + +export const getSingleDoc = (store, id) => { + return new Promise((resolve, reject) => { + const startQ = performance.now() + const request = store.get(id) + + request.onsuccess = event => { + const doc = event.target.result + const endQ = performance.now() + console.log(`Single data took : ${endQ - startQ} ms`) + resolve(doc) + } + + request.onerror = event => { + console.error('Error getting data:', event.target.error) + reject(event.target.error) + } + }) +} + +export const createIDBIndex = async ( + queryEngine, + { indexName, indexedFields, shouldRecreateIndex = false } +) => { + const dbName = queryEngine.db.name + // See https://github.com/pouchdb/pouchdb/blob/f2c665a2a885437b9cea80dda62c02a93a137c1e/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js#L38C1-L40C75 + const newVersion = Math.pow(10, 13) * 2 + new Date().getTime() + + // We need to first close the db to upgrade it with index creation, with new db version + queryEngine.db.close() + + const indexedDB = + typeof window == 'object' ? window.indexedDB : self.indexedDB + const upgradeRequest = indexedDB.open(dbName, newVersion) + + return new Promise((resolve, reject) => { + upgradeRequest.onupgradeneeded = async event => { + const store = upgradeRequest.transaction.objectStore( + queryEngine.storeName + ) + + if (shouldRecreateIndex) { + // Useful for testing + try { + store.deleteIndex(indexName) + } catch (err) { + if (!isMissingIDBIndexError(err)) { + throw err + } + } + } + + let idx + const fields = indexedFields.map(field => `data.${field}`) // indexeddb adapter put data in doc.data + if (indexedFields.length === 1) { + idx = store.createIndex(indexName, fields[0], { unique: false }) + } else { + idx = store.createIndex(indexName, fields, { unique: false }) + } + resolve(idx) + } + + upgradeRequest.onsuccess = () => { + queryEngine.db = upgradeRequest.result + } + + upgradeRequest.onerror = err => { + logger.error(`Error during db upgrade: `, err) + reject(err) + } + }) +} diff --git a/packages/cozy-pouch-link/src/db/idb/getDocs.spec.js b/packages/cozy-pouch-link/src/db/idb/getDocs.spec.js new file mode 100644 index 0000000000..650f61f889 --- /dev/null +++ b/packages/cozy-pouch-link/src/db/idb/getDocs.spec.js @@ -0,0 +1,414 @@ +import 'fake-indexeddb/auto' +import { findData } from './getDocs' + +let db +let store + +global.structuredClone = val => JSON.parse(JSON.stringify(val)) + +const openDB = async (dbName, version) => { + return new Promise((resolve, reject) => { + const req = indexedDB.open(dbName, version) + req.onsuccess = event => { + const db = req.result + resolve(db) + } + // This event is only implemented in recent browsers + req.onupgradeneeded = event => { + // Save the IDBDatabase interface + const db = event.target.result + + // Create an objectStore for this database + const store = db.createObjectStore('files', { keyPath: 'id' }) + store.createIndex('by_type', 'type') + store.createIndex('by_size', 'size') + store.createIndex('by_tags', 'tags') + store.createIndex('by_type_and_size', ['type', 'size']) + store.createIndex('by_type_and_size_and_name', ['type', 'size', 'name']) + } + }) +} + +describe('mangoToIDB', () => { + beforeEach(async () => { + db = await openDB('TestDB', '1') + + const docs = [ + { id: '1', type: 'pdf', size: 5000, name: 'docA', tags: ['work'] }, + { id: '2', type: 'pdf', size: 2000, name: 'docB' }, + { id: '3', type: 'doc', size: 7000, name: 'docC', tags: ['home'] }, + { id: '4', type: 'xls', size: 300, name: 'docD' }, + { id: '5', type: 'pdf', size: 1200, name: 'docE', tags: null } + ] + + const tx = db.transaction('files', 'readwrite') + store = tx.objectStore('files') + for (const doc of docs) { + store.put(doc) + } + await tx.done + }) + + afterEach(async () => { + db.close() + await indexedDB.deleteDatabase('TestDB') + }) + + it('should handle empty selector', async () => { + const docs = await findData(store, { + selector: {}, + indexedFields: null + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '2', '3', '4', '5']) + }) + + it('should handle explicit $eq', async () => { + const index = store.index('by_type') + const docs = await findData(index, { + selector: { type: { $eq: 'pdf' } }, + indexedFields: ['type'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '2', '5']) + }) + + it('should handle implicit $eq', async () => { + const index = store.index('by_type') + const docs = await findData(index, { + selector: { type: 'pdf' }, + indexedFields: ['type'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '2', '5']) + }) + + it('should handle $gt', async () => { + const index = store.index('by_size') + const docs = await findData(index, { + selector: { size: { $gt: 2000 } }, + indexedFields: ['size'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '3']) + }) + + it('should handle $gt null', async () => { + const index = store.index('by_size') + const docs = await findData(index, { + selector: { size: { $gt: null } }, + indexedFields: ['size'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '2', '3', '4', '5']) + }) + + it('should handle $gt null on multiple attributes', async () => { + const index = store.index('by_type_and_size') + const docs = await findData(index, { + selector: { type: { $gt: null }, size: { $gt: null } }, + indexedFields: ['type', 'size'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '2', '3', '4', '5']) + }) + + it('should handle $gt null among multiple attributes', async () => { + const index = store.index('by_type_and_size') + const docs = await findData(index, { + selector: { type: { $gt: null }, size: { $eq: 2000 } }, + indexedFields: ['type', 'size'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['2']) + }) + + it('should handle $lt', async () => { + const index = store.index('by_size') + const docs = await findData(index, { + selector: { size: { $lt: 2000 } }, + indexedFields: ['size'] + }) + expect(docs.map(d => d.id)).toEqual(['4', '5']) + }) + + it('should handle $and explicit', async () => { + const index = store.index('by_size') + const docs = await findData(index, { + selector: { + $and: [{ size: { $lte: 5000 } }, { size: { $gt: 1000 } }] + }, + indexedFields: ['size'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '2', '5']) + }) + + it('should handle $and implicit', async () => { + const index = store.index('by_size') + const docs = await findData(index, { + selector: { + size: { $gt: 1000, $lte: 5000 } + }, + indexedFields: ['size'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '2', '5']) + }) + + it('should handle multiple range conditions on same attribute', async () => { + const index = store.index('by_size') + let docs = await findData(index, { + selector: { + size: { $gt: 300, $lt: 2000 } + }, + indexedFields: ['size'] + }) + expect(docs.map(d => d.id)).toEqual(['5']) + + docs = await findData(index, { + selector: { + size: { $gt: 300, $lte: 2000 } + }, + indexedFields: ['size'] + }) + expect(docs.map(d => d.id)).toEqual(['5', '2']) + + docs = await findData(index, { + selector: { + size: { $gte: 300, $lte: 2000 } + }, + indexedFields: ['size'] + }) + expect(docs.map(d => d.id)).toEqual(['4', '5', '2']) + + docs = await findData(index, { + selector: { + size: { $gte: 300, $lt: 2000 } + }, + indexedFields: ['size'] + }) + expect(docs.map(d => d.id)).toEqual(['4', '5']) + }) + + it('should handle multiple range conditions on several attributes', async () => { + const index = store.index('by_type_and_size_and_name') + const docs = await findData(index, { + selector: { + type: { $gt: 'pdf' }, + size: { $gt: 300 }, + name: { $gt: 'docB' } + }, + indexedFields: ['size', 'name', 'type'] + }) + expect(docs.map(d => d.id)).toEqual(['4']) + }) + + it('should support compound queries with 1 equality and 1 range', async () => { + let index = store.index('by_type_and_size') + let docs = await findData(index, { + selector: { + type: 'pdf', + size: { $gte: 2000, $lte: 5000 } + }, + indexedFields: ['type', 'size'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '2']) + }) + + it('should support compound queries with 1 equality and multiple range', async () => { + let index = store.index('by_type_and_size') + let docs = await findData(index, { + selector: { + type: 'pdf', + size: { $gte: 2000, $lte: 5000 }, + name: { $gte: 'docB' } + }, + indexedFields: ['type', 'size'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['2']) + }) + + it('should support compound queries with multiple equalities', async () => { + const index = store.index('by_type_and_size_and_name') + + const docs = await findData(index, { + selector: { + type: 'pdf', + size: 2000, + name: 'docB' + }, + indexedFields: ['type', 'size', 'name'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['2']) + }) + + it('should support compound queries with several ranges', async () => { + const index = store.index('by_type_and_size') + const docs = await findData(index, { + selector: { + type: { $gt: 'pdf', $lt: 'zzzz' }, + size: { $gte: 300, $lte: 1200 } + }, + indexedFields: ['type', 'size'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['4', '5']) + }) + + it('should support compound queries with missing field in selector', async () => { + const index = store.index('by_type_and_size') + const docs = await findData(index, { + selector: { + type: { $gt: 'pdf' } + }, + indexedFields: ['type', 'size'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '2', '4', '5']) + }) + + it('should not return docs for malformed compound queries', async () => { + const index = store.index('by_type_and_size') + let docs = await findData(index, { + selector: { + type: 'pdf', + size: { $gte: 2000, $lte: 5000 } + }, + indexedFields: ['size', 'type'], // reversed indexed fields + index + }) + expect(docs.map(d => d.id).sort()).toEqual([]) + }) + + it('should handle $or on indexed attributes', async () => { + const index = store.index('by_type_and_size') + const docs = await findData(index, { + selector: { + $or: [{ type: 'xls' }, { size: 2000 }] + }, + index, + indexedFields: ['type', 'size'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['2', '4']) + }) + + it('should handle $or on non-indexed attributes', async () => { + const index = store.index('by_type') + const docs = await findData(index, { + selector: { + $or: [{ type: 'xls' }, { name: 'docC' }] + }, + index, + indexedFields: ['type'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['3', '4']) + }) + + it('should handle $in', async () => { + const index = store.index('by_type') + const docs = await findData(index, { + selector: { type: { $in: ['pdf', 'xls'] } }, + indexedFields: ['type'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '2', '4', '5']) + }) + + it('should handle $exists: true', async () => { + const index = store.index('by_tags') + const docs = await findData(index, { + selector: { tags: { $exists: true } }, + indexedFields: ['tags'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['1', '3']) + }) + + it('should handle $exists: false', async () => { + const docs = await findData(store, { + selector: { tags: { $exists: false } }, + indexedFields: ['tags'], + index: store + }) + expect(docs.map(d => d.id).sort()).toEqual(['2', '4']) + }) + + it('should handle $ne', async () => { + const index = store.index('by_type') + const docs = await findData(index, { + selector: { type: { $ne: 'pdf' } }, + indexedFields: ['type'] + }) + expect(docs.map(d => d.id).sort()).toEqual(['3', '4']) + }) + + it('should correctly sort asc', async () => { + const index = store.index('by_size') + const docs = await findData(index, { + selector: { size: { $gte: 300, $lte: 2000 } }, + indexedFields: ['size'], + sort: [{ size: 'asc' }] + }) + expect(docs.map(d => d.id)).toEqual(['4', '5', '2']) + }) + + it('should correctly sort desc', async () => { + const index = store.index('by_size') + const docs = await findData(index, { + selector: { size: { $gte: 300, $lte: 2000 } }, + indexedFields: ['size'], + sort: [{ size: 'desc' }] + }) + expect(docs.map(d => d.id)).toEqual(['2', '5', '4']) + }) + + it('should correctly sort non-indexed fields asc', async () => { + const index = store.index('by_type') + const docs = await findData(index, { + selector: { size: { $gte: 2000, $lte: 7000 } }, + indexedFields: ['type'], + sort: [{ size: 'asc' }] + }) + expect(docs.map(d => d.id)).toEqual(['2', '1', '3']) + }) + + it('should correctly sort non-indexed fields desc', async () => { + const index = store.index('by_type') + const docs = await findData(index, { + selector: { size: { $gte: 2000, $lte: 7000 } }, + indexedFields: ['type'], + sort: [{ size: 'desc' }] + }) + expect(docs.map(d => d.id)).toEqual(['3', '1', '2']) + }) + + it('should throw on unsupported operators', async () => { + const index = store.index('by_type') + await expect( + findData(index, { + selector: { type: { $wrongOpe: 'test' } }, + indexedFields: ['type'], + index + }) + ).rejects.toThrow('Unsupported operation: $wrongOpe') + }) + + it('should handle limit', async () => { + const index = store.index('by_type') + const docs = await findData(index, { + selector: { type: 'pdf' }, + indexedFields: ['type'], + limit: 2 + }) + expect(docs.map(d => d.id)).toEqual(['1', '2']) + }) + + it('should handle offset', async () => { + const index = store.index('by_type') + const docs = await findData(index, { + selector: { type: 'pdf' }, + indexedFields: ['type'], + offset: 2 + }) + expect(docs.map(d => d.id)).toEqual(['5']) + }) + + it('should handle limit + offset', async () => { + const index = store.index('by_type') + const docs = await findData(index, { + selector: { type: 'pdf' }, + indexedFields: ['type'], + offset: 1, + limit: 1 + }) + expect(docs.map(d => d.id)).toEqual(['2']) + }) +}) diff --git a/packages/cozy-pouch-link/src/db/idb/idb.js b/packages/cozy-pouch-link/src/db/idb/idb.js new file mode 100644 index 0000000000..68b055d7b8 --- /dev/null +++ b/packages/cozy-pouch-link/src/db/idb/idb.js @@ -0,0 +1,167 @@ +import DatabaseQueryEngine from '../dbInterface' +import { getIndexFields, getIndexName } from '../../mango' +import logger from '../../logger' +import { + createIDBIndex, + findData, + getAllData, + getSingleDoc, + MAX_LIMIT, + parseResults +} from './getDocs' +import { isMissingIDBIndexError } from '../../errors' + +export default class IndexedDBQueryEngine extends DatabaseQueryEngine { + constructor(pouchManager, doctype) { + super() + this.db = null + this.client = pouchManager?.client + this.doctype = doctype + this.storeName = `docs` + this.openRequest + } + + openDB(dbName, version = undefined, { forceName = false } = {}) { + const indexedDB = + typeof window == 'object' ? window.indexedDB : self.indexedDB + const fullDbName = forceName ? dbName : `_pouch_${dbName}` + const request = version + ? indexedDB.open(fullDbName, version) + : indexedDB.open(fullDbName) + + request.onsuccess = event => { + const db = request.result + this.db = db + this.openRequest = request + } + request.onerror = event => { + logger.error(`Database error: ${event.target.error}`) + } + } + + async allDocs({ limit = MAX_LIMIT, skip = 0 } = {}) { + try { + const tx = this.db.transaction(this.storeName, 'readonly') + const store = tx.objectStore(this.storeName) + const docs = await getAllData(store, { limit, skip }) + const resp = parseResults(this.client, docs, this.doctype, { + limit, + skip + }) + return resp + } catch (err) { + logger.error(err) + return null + } + } + + async getById(id) { + try { + if (!id) { + return { data: null } + } + const tx = this.db.transaction(this.storeName, 'readonly') + const store = tx.objectStore(this.storeName) + const doc = await getSingleDoc(store, id) + const resp = parseResults(this.client, [doc], this.doctype) + return resp + } catch (err) { + logger.error(err) + return null + } + } + + async getByIds(ids) { + try { + const tx = this.db.transaction(this.storeName, 'readonly') + const store = tx.objectStore(this.storeName) + const docs = await Promise.all(ids.map(id => store.get(id))) + const resp = parseResults( + this.client, + docs.filter(doc => doc !== undefined), + this.doctype + ) + return resp + } catch (err) { + logger.error(err) + return null + } + } + + async find(options) { + const { + selector, + sort, + limit, + partialFilter, + skip = 0, + shouldRecreateIndex = false + } = options + let { indexedFields, recreateIndex } = options + + indexedFields = getIndexFields({ + indexedFields, + selector, + sort, + partialFilter + }) + const indexName = getIndexName({ + selector, + sort, + partialFilter, + indexedFields + }) + const findOpts = { + selector, + sort, + partialFilter, + offset: skip, + limit, + doctype: this.doctype, + use_index: indexName, + ...options + } + findOpts.indexedFields = indexedFields + + try { + const tx = this.db.transaction(this.storeName, 'readonly') + const store = tx.objectStore(this.storeName) + + if (recreateIndex === true) { + throw new Error('The specified index was not found') + } + + const index = store.index(indexName) + if (index && shouldRecreateIndex) { + // Useful for testing + throw new Error('The specified index was not found') + } + + const docs = await findData(index, findOpts) + const resp = parseResults(this.client, docs, this.doctype, { + limit, + skip + }) + + return resp + } catch (err) { + if (isMissingIDBIndexError(err)) { + logger.info('Missing index, create it...') + + const newIndex = await createIDBIndex(this, { + indexName, + indexedFields, + shouldRecreateIndex + }) + const docs = await findData(newIndex, findOpts) + const resp = parseResults(this.client, docs, this.doctype, { + limit, + skip + }) + return resp + } + logger.error(err) + return null + } + } +} diff --git a/packages/cozy-pouch-link/src/db/idb/mango.js b/packages/cozy-pouch-link/src/db/idb/mango.js new file mode 100644 index 0000000000..0d3d4bdef3 --- /dev/null +++ b/packages/cozy-pouch-link/src/db/idb/mango.js @@ -0,0 +1,368 @@ +import sift from 'sift' +import isPlainObject from 'lodash/isPlainObject' +import { MAX_LIMIT, queryWithAll, queryWithCursor } from './getDocs' +import orderBy from 'lodash/orderBy' + +const IDB_ASC_ORDER = 'next' +const IDB_DESC_ORDER = 'prev' + +const RANGE_OP = ['$gt', '$lt', '$gte', '$lte'] +const SUPPORTED_CURSOR_OP = [...RANGE_OP, '$eq', '$in'] + +export const extractFiltersFromSelector = ( + selector, + indexedFields, + { forceInMemory = false } = {} +) => { + const rangeFields = [] + const notCursorFields = [] + + // XXX - Here we extract from the selector the predicates that can evaluated through an IDB cursor, + // and those that should be evaluated directly in-memory. + for (const [field, condition] of Object.entries(selector)) { + if (typeof condition === 'object' && condition !== null) { + const ops = Object.keys(condition) + if (ops.some(op => RANGE_OP.includes(op))) { + // Multiple range queries are badly handled by cursors + rangeFields.push(field) + } + if (ops.some(op => !SUPPORTED_CURSOR_OP.includes(op))) { + // Not supported operators by cursor + notCursorFields.push(field) + } + if ( + ops.some( + (op, i) => op === '$gt' && Object.values(condition)[i] === null + ) + ) { + // $gt: null evaluation is not supported by cursors, and there is no generic value to express + // "any data" whatever the type. So let's exclude it from cursor evaluation + // TODO if at least one attribute cannot be evaluated with cursor, all should be in-memory, otherwise + // the index cannot be used correctly + console.log('GT NULL') + notCursorFields.push(field) + } + } + } + const firstField = indexedFields[0] + const firstFieldHasRange = rangeFields.includes(firstField) + + let notForCursorFilters = {} + let cursorFilters = {} + + if (notCursorFields.length > 0) { + notForCursorFilters = selector + return { notForCursorFilters, cursorFilters } + } + + for (const [field, condition] of Object.entries(selector)) { + const isIndexed = indexedFields.includes(field) + const indexPos = indexedFields.indexOf(field) + const hasRange = rangeFields.includes(field) + const notCursorOp = notCursorFields.includes(field) + const isNotCursorFriendly = + !isIndexed || + (hasRange && indexPos > 0 && firstFieldHasRange) || + notCursorOp + + if (isNotCursorFriendly || forceInMemory) { + notForCursorFilters[field] = condition + } else { + cursorFilters[field] = condition + } + } + + return { + notForCursorFilters, + cursorFilters + } +} + +const sortInMemory = (docs, sort) => { + const attributes = sort.map(s => Object.keys(s)[0]) + const orders = sort.map(s => Object.values(s)[0]) + return orderBy(docs, attributes, orders) +} + +export const getSortDirection = sort => { + if (!sort || sort.length < 1) { + return IDB_ASC_ORDER // default direction + } + const sortDir = Object.values(sort[0])[0] + return sortDir === 'asc' ? IDB_ASC_ORDER : IDB_DESC_ORDER +} + +// Adapted from cozy-client +const convert$gtNullSelectors = selector => { + for (const [key, value] of Object.entries(selector)) { + const convertedValue = isPlainObject(value) + ? convert$gtNullSelectors(value) + : value + + if ( + (key === '$gt' && convertedValue === null) || + (key === '$gt' && convertedValue === '') + ) { + delete selector[key] + selector['$exists'] = true + } + } + + return selector +} + +export const evaluateSelectorInMemory = async (store, data, selector, sort) => { + let results = data + if (!data || data.length < 1) { + // In case there is no given data, query all of them + results = await queryWithAll(store) + results.map(res => ({ _id: res.id, _rev: res.rev, ...res })) + } + // sift does not work like couchdb when using { $gt: null } as a selector, so we convert the operator + const convertedSelector = convert$gtNullSelectors(selector) + const evaluator = sift(convertedSelector) + + if (sort) { + return sortInMemory(results.filter(evaluator), sort) + } + return results.filter(evaluator) +} + +const mergeExplicitAndConditions = conditions => { + const result = {} + + conditions.forEach(condition => { + for (const key in condition) { + if (!result[key]) { + result[key] = {} + } + Object.assign(result[key], condition[key]) + } + }) + + return result +} + +const createMergedIDBKeyRange = ({ conditions, fields }) => { + const lower = [] + let upper = [] + let lowerOpen = false, + upperOpen = false + + fields.forEach((field, idx) => { + let condition = conditions[field] + + if (!condition) { + return + } + + if (typeof condition !== 'object') { + // Special case for implicit equality + condition = { $eq: conditions[field] } + } + let fieldLower, + fieldUpper, + fieldLowerOpen = false, + fieldUpperOpen = false + + for (const [op, value] of Object.entries(condition)) { + switch (op) { + case '$gt': + if (fieldLower === undefined || value > fieldLower) { + fieldLower = value + fieldLowerOpen = true + } + break + case '$gte': + if ( + fieldLower === undefined || + value > fieldLower || + (value === fieldLower && fieldLowerOpen) + ) { + fieldLower = value + fieldLowerOpen = false + } + break + case '$lt': + if (fieldUpper === undefined || value < fieldUpper) { + fieldUpper = value + fieldUpperOpen = true + } + break + case '$lte': + if ( + fieldUpper === undefined || + value < fieldUpper || + (value === fieldUpper && fieldUpperOpen) + ) { + fieldUpper = value + fieldUpperOpen = false + } + break + case '$eq': + fieldLower = value + fieldUpper = value + fieldLowerOpen = false + fieldUpperOpen = false + break + case '$in': + case '$exists': + case '$ne': + break + default: + throw new Error(`Operator not supported: ${op}`) + } + } + + lower.push(fieldLower) + upper.push(fieldUpper) + lowerOpen = lowerOpen || fieldLowerOpen + upperOpen = upperOpen || fieldUpperOpen + }) + if (lower.every(v => v === undefined) && upper.every(v => v === undefined)) { + return null + } + + if (fields.length === 1) { + // Single-field index + if ( + lower[0] !== undefined && + upper[0] !== undefined && + lower[0] === upper[0] + ) { + return IDBKeyRange.only(lower[0]) + } + if (lower[0] === null || upper[0] === null) { + // Special case to handle null range predicates, e.g. { $gt: null } + // IDB does not work on null values + return null + } + if (lower[0] !== undefined && upper[0] !== undefined) { + return IDBKeyRange.bound(lower[0], upper[0], lowerOpen, upperOpen) + } + if (lower[0] !== undefined) { + return IDBKeyRange.lowerBound(lower[0], lowerOpen) + } + if (upper[0] !== undefined) { + return IDBKeyRange.upperBound(upper[0], upperOpen) + } + } + + if (lower[0] === undefined) { + return IDBKeyRange.upperBound(upper, upperOpen) + } + if (upper[0] === undefined) { + return IDBKeyRange.lowerBound(lower, lowerOpen) + } + return IDBKeyRange.bound(lower, upper, lowerOpen, upperOpen) +} + +const queryByField = async ( + index, + fields, + conditions, + sort, + { offset = 0, limit = Infinity, forceInMemory = false } = {} +) => { + const sortDirection = getSortDirection(sort) + + const { cursorFilters, notForCursorFilters } = extractFiltersFromSelector( + conditions, + fields, + { forceInMemory } + ) + let allResults = [] + + for (const attribute of Object.keys(cursorFilters)) { + const filter = cursorFilters[attribute] + if (typeof filter === 'object' && '$in' in filter) { + let results = [] + for (const val of filter['$in']) { + const idbKeyRange = IDBKeyRange.only(val) + const res = await queryWithCursor(index, idbKeyRange, { + offset, + limit, + sortDirection + }) + if (res && res.length > 0) { + results = results.concat(res) + } + } + allResults = Array.from(results) + delete cursorFilters[attribute] + } + } + if (Object.keys(cursorFilters).length > 0) { + const idbKeyRange = createMergedIDBKeyRange({ + conditions: cursorFilters, + fields: Array.isArray(fields) ? fields : [fields] + }) + const results = await queryWithCursor(index, idbKeyRange, { + offset, + limit, + sortDirection + }) + allResults = allResults.concat(results) + } + + if (notForCursorFilters && Object.keys(notForCursorFilters).length > 0) { + allResults = await evaluateSelectorInMemory( + index, + allResults, + notForCursorFilters, + sort + ) + } + return allResults +} + +export const executeIDBFind = async ( + store, + { + selector, + indexedFields, + sort, + forceInMemory = false, + limit = MAX_LIMIT, + offset = 0 + } +) => { + if ('$or' in selector) { + const subQueries = await Promise.all( + selector['$or'].map(cond => + executeIDBFind(store, { + selector: cond, + indexedFields, + sort, + limit, + offset, + forceInMemory: true + }) + ) + ) + const seen = new Set() + return subQueries.flat().filter(doc => { + const key = doc.id || JSON.stringify(doc) + if (seen.has(key)) return false + seen.add(key) + return true + }) + } + + let evalSelector = selector + if ('$and' in selector) { + evalSelector = mergeExplicitAndConditions(selector['$and']) + } + + if (Object.keys(evalSelector).length > 0) { + return queryByField(store, indexedFields, evalSelector, sort, { + offset, + limit, + forceInMemory + }) + } + + // fallback for empty selector + const sortDirection = getSortDirection(sort) + return queryWithCursor(store, null, { offset, limit, sortDirection }) +} diff --git a/packages/cozy-pouch-link/src/db/idb/mango.spec.js b/packages/cozy-pouch-link/src/db/idb/mango.spec.js new file mode 100644 index 0000000000..9a8faf488c --- /dev/null +++ b/packages/cozy-pouch-link/src/db/idb/mango.spec.js @@ -0,0 +1,227 @@ +import { + extractFiltersFromSelector, + evaluateSelectorInMemory, + getSortDirection +} from './mango.js' + +describe('extractFiltersFromSelector', () => { + it('should detect no issues when only one range on first indexed field', () => { + const selector = { + type: { $gt: 'a' }, + status: { $eq: 'ok' } + } + const indexedFields = ['type', 'status'] + + const result = extractFiltersFromSelector(selector, indexedFields) + + expect(result).toEqual({ + notForCursorFilters: {}, + cursorFilters: { + type: { $gt: 'a' }, + status: { $eq: 'ok' } + } + }) + }) + + it('should detect a second range on a non-first indexed field', () => { + const selector = { + type: { $gt: 'a' }, + size: { $lt: 1000 } + } + const indexedFields = ['type', 'size'] + + const result = extractFiltersFromSelector(selector, indexedFields) + + expect(result).toEqual({ + notForCursorFilters: { + size: { $lt: 1000 } + }, + cursorFilters: { + type: { $gt: 'a' } + } + }) + }) + + it('should detect range on non-indexed field', () => { + const selector = { + type: { $gt: 'a' }, + date: { $lt: '2024-01-01' } + } + const indexedFields = ['type'] + + const result = extractFiltersFromSelector(selector, indexedFields) + + expect(result).toEqual({ + notForCursorFilters: { + date: { $lt: '2024-01-01' } + }, + cursorFilters: { + type: { $gt: 'a' } + } + }) + }) + + it('should group multiple non-indexed range fields', () => { + const selector = { + foo: { $gt: 1 }, + bar: { $lt: 10 }, + baz: { $eq: 'test' } + } + const indexedFields = ['baz'] + + const result = extractFiltersFromSelector(selector, indexedFields) + + expect(result).toEqual({ + notForCursorFilters: { + foo: { $gt: 1 }, + bar: { $lt: 10 } + }, + cursorFilters: { + baz: { $eq: 'test' } + } + }) + }) + + it('should detect non-indexed range field', () => { + const selector = { + baz: { $eq: 'test' }, + foo: { $gt: 1 }, + bar: { $lt: 10 } + } + const indexedFields = ['baz', 'foo'] + + const result = extractFiltersFromSelector(selector, indexedFields) + + expect(result).toEqual({ + notForCursorFilters: { + bar: { $lt: 10 } + }, + cursorFilters: { + baz: { $eq: 'test' }, + foo: { $gt: 1 } + } + }) + }) + + it('should allow simple equality conditions for cursor', () => { + const selector = { + type: 'pdf', + category: 'report' + } + const indexedFields = ['type', 'category'] + + const result = extractFiltersFromSelector(selector, indexedFields) + + expect(result).toEqual({ + notForCursorFilters: {}, + cursorFilters: { + type: 'pdf', + category: 'report' + } + }) + }) + + it('should allow cursor for one range and multiple equalities', () => { + const selector = { + type: 'pdf', + category: 'report', + date: { $gt: '2024-01-01' } + } + const indexedFields = ['type', 'category', 'date'] + + const result = extractFiltersFromSelector(selector, indexedFields) + + expect(result).toEqual({ + notForCursorFilters: {}, + cursorFilters: { + type: 'pdf', + category: 'report', + date: { $gt: '2024-01-01' } + } + }) + }) + + it('should force non-cursor with forceInMemory option', () => { + const selector = { + type: 'pdf' + } + const indexedFields = ['type'] + const result = extractFiltersFromSelector(selector, indexedFields, { + forceInMemory: true + }) + + expect(result).toEqual({ + notForCursorFilters: { type: 'pdf' }, + cursorFilters: {} + }) + }) +}) + +describe('evaluateSelectorInMemory', () => { + const data = [ + { + _id: '1', + name: 'a', + size: 10, + type: 'type1' + }, + { + _id: '2', + name: 'b', + size: 500, + type: 'type1' + }, + { + _id: '3', + name: 'c', + size: 100, + type: 'type2' + }, + { + _id: '4', + name: 'd', + size: 200, + type: 'type2' + } + ] + it('should return all data with null selector', async () => { + const selector = { _id: { $gt: null } } + expect(await evaluateSelectorInMemory(null, data, selector)).toEqual(data) + }) + + it('should correctly apply range ope', async () => { + const selector = { size: { $gt: 100, $lt: 500 } } + expect(await evaluateSelectorInMemory(null, data, selector)).toEqual([ + data[3] + ]) + }) + + it('should correctly apply equality ope', async () => { + const selector = { size: { $eq: 100 } } + expect(await evaluateSelectorInMemory(null, data, selector)).toEqual([ + data[2] + ]) + }) + + it('should deal with mulitple operators', async () => { + const selector = { type: 'type1', size: { $gt: 100 } } + expect(await evaluateSelectorInMemory(null, data, selector)).toEqual([ + data[1] + ]) + }) +}) + +describe('sort direction', () => { + it('should get ascending order', () => { + const sort = [{ date: 'asc', name: 'asc' }] + expect(getSortDirection(sort)).toEqual('next') + }) + it('should get desc order', () => { + const sort = [{ date: 'desc', name: 'desc' }] + expect(getSortDirection(sort)).toEqual('prev') + }) + it('should get asc order as default', () => { + const sort = undefined + expect(getSortDirection(sort)).toEqual('next') + }) +}) diff --git a/packages/cozy-pouch-link/src/errors.js b/packages/cozy-pouch-link/src/errors.js index 2ca56aed14..c41dd6262f 100644 --- a/packages/cozy-pouch-link/src/errors.js +++ b/packages/cozy-pouch-link/src/errors.js @@ -5,6 +5,7 @@ const INVALID_TOKEN_ALT_ERROR = /Invalid token/ const SQLITE_MISSING_INDEX_ERROR = /no such index/ const POUCHDB_MISSING_INDEX_ERROR = /Could not find that index/ const POUCHDB_MISSING_INDEX_ERROR_ALT = /no index/ +const IDB_MISSING_INDEX_ERROR = /The specified index was not found/ const expiredTokenError = error => { const errorMsg = error.message @@ -38,3 +39,7 @@ export const isMissingPouchDBIndexError = error => { POUCHDB_MISSING_INDEX_ERROR_ALT.test(error.message) ) } + +export const isMissingIDBIndexError = error => { + return IDB_MISSING_INDEX_ERROR.test(error.message) +} diff --git a/packages/cozy-pouch-link/src/index.js b/packages/cozy-pouch-link/src/index.js index 925e15f53e..5ecd4ceab7 100644 --- a/packages/cozy-pouch-link/src/index.js +++ b/packages/cozy-pouch-link/src/index.js @@ -1,2 +1,3 @@ export { default } from './CozyPouchLink' export { default as SQLiteQuery } from './db/sqlite/sqliteDb' +export { default as IndexedDBQuery } from './db/idb/idb' diff --git a/yarn.lock b/yarn.lock index 59d66e100b..72efb1f936 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6681,6 +6681,7 @@ __metadata: "@op-engineering/op-sqlite": "npm:^11.4.8" cozy-client: "npm:^57.7.0" cozy-device-helper: "npm:2.7.0" + fake-indexeddb: "npm:^6.0.0" jest-localstorage-mock: "npm:2.4.19" parcel: "npm:2.13.3" pouchdb-adapter-memory: "npm:7.2.2" @@ -6688,6 +6689,7 @@ __metadata: pouchdb-find: "npm:^7.2.2" react: "npm:16.14.0" react-dom: "npm:16.14.0" + sift: "npm:^17.1.3" typescript: "npm:4.1.5" peerDependencies: "@cozy/minilog": 1.0.0 @@ -8513,6 +8515,13 @@ __metadata: languageName: node linkType: hard +"fake-indexeddb@npm:^6.0.0": + version: 6.0.0 + resolution: "fake-indexeddb@npm:6.0.0" + checksum: 10c0/551ddada98f6d8b1c159698ff82570d06c8959f8e98616d872305dee3d857795dd62dd32094779e7b325397103364dcb0e24b6447992c0464ae13d19fbe57315 + languageName: node + linkType: hard + "fast-deep-equal@npm:^3.1.1, fast-deep-equal@npm:^3.1.3": version: 3.1.3 resolution: "fast-deep-equal@npm:3.1.3" @@ -16618,6 +16627,13 @@ __metadata: languageName: node linkType: hard +"sift@npm:^17.1.3": + version: 17.1.3 + resolution: "sift@npm:17.1.3" + checksum: 10c0/bb05d1d65cc9b549b402c1366ba1fcf685311808b6d5c2f4fa2f477d7b524218bbf6c99587562d5613d407820a6b5a7cad809f89c3f75c513ff5d8c0e0a0cead + languageName: node + linkType: hard + "sift@npm:^6.0.0": version: 6.0.0 resolution: "sift@npm:6.0.0" From fd2fe59829659cf88219b7c60c2ac969a13c890d Mon Sep 17 00:00:00 2001 From: Paul Tran-Van Date: Thu, 24 Apr 2025 11:25:14 +0200 Subject: [PATCH 7/7] docs: Add jsdoc and types --- docs/api/cozy-pouch-link/README.md | 1 + .../cozy-pouch-link/classes/IndexedDBQuery.md | 205 ++++++++++++++++++ packages/cozy-pouch-link/src/db/idb/idb.js | 2 +- .../cozy-pouch-link/types/db/dbInterface.d.ts | 12 +- .../cozy-pouch-link/types/db/helpers.d.ts | 1 + .../cozy-pouch-link/types/db/idb/getDocs.d.ts | 37 ++++ .../cozy-pouch-link/types/db/idb/idb.d.ts | 9 + .../cozy-pouch-link/types/db/idb/mango.d.ts | 16 ++ .../cozy-pouch-link/types/db/sqlite/sql.d.ts | 1 - packages/cozy-pouch-link/types/errors.d.ts | 1 + packages/cozy-pouch-link/types/index.d.ts | 1 + 11 files changed, 278 insertions(+), 8 deletions(-) create mode 100644 docs/api/cozy-pouch-link/classes/IndexedDBQuery.md create mode 100644 packages/cozy-pouch-link/types/db/idb/getDocs.d.ts create mode 100644 packages/cozy-pouch-link/types/db/idb/idb.d.ts create mode 100644 packages/cozy-pouch-link/types/db/idb/mango.d.ts diff --git a/docs/api/cozy-pouch-link/README.md b/docs/api/cozy-pouch-link/README.md index 76a17cc9db..00ac1a4265 100644 --- a/docs/api/cozy-pouch-link/README.md +++ b/docs/api/cozy-pouch-link/README.md @@ -4,5 +4,6 @@ cozy-pouch-link ## Classes +* [IndexedDBQuery](classes/IndexedDBQuery.md) * [PouchLink](classes/PouchLink.md) * [SQLiteQuery](classes/SQLiteQuery.md) diff --git a/docs/api/cozy-pouch-link/classes/IndexedDBQuery.md b/docs/api/cozy-pouch-link/classes/IndexedDBQuery.md new file mode 100644 index 0000000000..8249fe930c --- /dev/null +++ b/docs/api/cozy-pouch-link/classes/IndexedDBQuery.md @@ -0,0 +1,205 @@ +[cozy-pouch-link](../README.md) / IndexedDBQuery + +# Class: IndexedDBQuery + +## Hierarchy + +* `DatabaseQueryEngine` + + ↳ **`IndexedDBQuery`** + +## Constructors + +### constructor + +• **new IndexedDBQuery**(`pouchManager`, `doctype`) + +*Parameters* + +| Name | Type | +| :------ | :------ | +| `pouchManager` | `any` | +| `doctype` | `any` | + +*Overrides* + +DatabaseQueryEngine.constructor + +*Defined in* + +[db/idb/idb.js:15](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/db/idb/idb.js#L15) + +## Properties + +### client + +• **client**: `any` + +*Defined in* + +[db/idb/idb.js:18](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/db/idb/idb.js#L18) + +*** + +### db + +• **db**: `IDBDatabase` + +*Defined in* + +[db/idb/idb.js:17](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/db/idb/idb.js#L17) + +*** + +### doctype + +• **doctype**: `any` + +*Defined in* + +[db/idb/idb.js:19](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/db/idb/idb.js#L19) + +*** + +### openRequest + +• **openRequest**: `IDBOpenDBRequest` + +*Defined in* + +[db/idb/idb.js:35](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/db/idb/idb.js#L35) + +*** + +### storeName + +• **storeName**: `string` + +*Defined in* + +[db/idb/idb.js:20](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/db/idb/idb.js#L20) + +## Methods + +### allDocs + +▸ **allDocs**(`__namedParameters?`): `Promise`<{ `data`: `any` = doc; `meta`: `undefined` ; `next`: `undefined` ; `skip`: `undefined` } | { `data`: `any`\[] = parsedResults; `meta`: { `count`: `number` = parsedResults.length } ; `next`: `boolean` ; `skip`: `number` }> + +*Parameters* + +| Name | Type | +| :------ | :------ | +| `__namedParameters` | `Object` | +| `__namedParameters.limit` | `number` | +| `__namedParameters.skip` | `number` | + +*Returns* + +`Promise`<{ `data`: `any` = doc; `meta`: `undefined` ; `next`: `undefined` ; `skip`: `undefined` } | { `data`: `any`\[] = parsedResults; `meta`: { `count`: `number` = parsedResults.length } ; `next`: `boolean` ; `skip`: `number` }> + +*Overrides* + +DatabaseQueryEngine.allDocs + +*Defined in* + +[db/idb/idb.js:42](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/db/idb/idb.js#L42) + +*** + +### find + +▸ **find**(`options`): `Promise`<{ `data`: `any` = doc; `meta`: `undefined` ; `next`: `undefined` ; `skip`: `undefined` } | { `data`: `any`\[] = parsedResults; `meta`: { `count`: `number` = parsedResults.length } ; `next`: `boolean` ; `skip`: `number` }> + +*Parameters* + +| Name | Type | +| :------ | :------ | +| `options` | `any` | + +*Returns* + +`Promise`<{ `data`: `any` = doc; `meta`: `undefined` ; `next`: `undefined` ; `skip`: `undefined` } | { `data`: `any`\[] = parsedResults; `meta`: { `count`: `number` = parsedResults.length } ; `next`: `boolean` ; `skip`: `number` }> + +*Overrides* + +DatabaseQueryEngine.find + +*Defined in* + +[db/idb/idb.js:91](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/db/idb/idb.js#L91) + +*** + +### getById + +▸ **getById**(`id`): `Promise`<{ `data`: `any` = doc; `meta`: `undefined` ; `next`: `undefined` ; `skip`: `undefined` } | { `data`: `any`\[] = parsedResults; `meta`: { `count`: `number` = parsedResults.length } ; `next`: `boolean` ; `skip`: `number` }> + +*Parameters* + +| Name | Type | +| :------ | :------ | +| `id` | `any` | + +*Returns* + +`Promise`<{ `data`: `any` = doc; `meta`: `undefined` ; `next`: `undefined` ; `skip`: `undefined` } | { `data`: `any`\[] = parsedResults; `meta`: { `count`: `number` = parsedResults.length } ; `next`: `boolean` ; `skip`: `number` }> + +*Overrides* + +DatabaseQueryEngine.getById + +*Defined in* + +[db/idb/idb.js:58](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/db/idb/idb.js#L58) + +*** + +### getByIds + +▸ **getByIds**(`ids`): `Promise`<{ `data`: `any` = doc; `meta`: `undefined` ; `next`: `undefined` ; `skip`: `undefined` } | { `data`: `any`\[] = parsedResults; `meta`: { `count`: `number` = parsedResults.length } ; `next`: `boolean` ; `skip`: `number` }> + +*Parameters* + +| Name | Type | +| :------ | :------ | +| `ids` | `any` | + +*Returns* + +`Promise`<{ `data`: `any` = doc; `meta`: `undefined` ; `next`: `undefined` ; `skip`: `undefined` } | { `data`: `any`\[] = parsedResults; `meta`: { `count`: `number` = parsedResults.length } ; `next`: `boolean` ; `skip`: `number` }> + +*Overrides* + +DatabaseQueryEngine.getByIds + +*Defined in* + +[db/idb/idb.js:74](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/db/idb/idb.js#L74) + +*** + +### openDB + +▸ **openDB**(`dbName`, `version?`, `__namedParameters?`): `void` + +*Parameters* + +| Name | Type | Default value | +| :------ | :------ | :------ | +| `dbName` | `any` | `undefined` | +| `version` | `any` | `undefined` | +| `__namedParameters` | `Object` | `{}` | +| `__namedParameters.forceName` | `boolean` | `undefined` | + +*Returns* + +`void` + +*Overrides* + +DatabaseQueryEngine.openDB + +*Defined in* + +[db/idb/idb.js:24](https://github.com/cozy/cozy-client/blob/master/packages/cozy-pouch-link/src/db/idb/idb.js#L24) diff --git a/packages/cozy-pouch-link/src/db/idb/idb.js b/packages/cozy-pouch-link/src/db/idb/idb.js index 68b055d7b8..609f845694 100644 --- a/packages/cozy-pouch-link/src/db/idb/idb.js +++ b/packages/cozy-pouch-link/src/db/idb/idb.js @@ -35,7 +35,7 @@ export default class IndexedDBQueryEngine extends DatabaseQueryEngine { this.openRequest = request } request.onerror = event => { - logger.error(`Database error: ${event.target.error}`) + logger.error(`Database error: ${JSON.stringify(event.target)}`) } } diff --git a/packages/cozy-pouch-link/types/db/dbInterface.d.ts b/packages/cozy-pouch-link/types/db/dbInterface.d.ts index 29c6754180..01b85ac8a0 100644 --- a/packages/cozy-pouch-link/types/db/dbInterface.d.ts +++ b/packages/cozy-pouch-link/types/db/dbInterface.d.ts @@ -97,21 +97,21 @@ declare class DatabaseQueryEngine { * Get all docs * * @param {AllDocsParams} options - The all docs options - * @returns {Promise} The found docs + * @returns {Promise} The found docs */ - allDocs(options: AllDocsParams): Promise; + allDocs(options: AllDocsParams): Promise; /** * Get a single doc by its id * * @param {string} id - id of the document to get - * @returns {Promise} The found docs + * @returns {Promise} The found docs */ - getById(id: string): Promise; + getById(id: string): Promise; /** * Get several docs by their ids * * @param {Array} ids - ids of the documents to get - * @returns {Promise} The found docs + * @returns {Promise} The found docs */ - getByIds(ids: Array): Promise; + getByIds(ids: Array): Promise; } diff --git a/packages/cozy-pouch-link/types/db/helpers.d.ts b/packages/cozy-pouch-link/types/db/helpers.d.ts index 5f5836dc7a..9668b15dee 100644 --- a/packages/cozy-pouch-link/types/db/helpers.d.ts +++ b/packages/cozy-pouch-link/types/db/helpers.d.ts @@ -1,4 +1,5 @@ export function getExistingDocument(queryEngine: DatabaseQueryEngine, id: string, throwIfNotFound?: boolean): Promise; export function areDocsEqual(oldDoc: any, newDoc: any): Promise; export function getCozyPouchData(doc: import('../CozyPouchLink').CozyPouchDocument): Array; +export function keepDocWitHighestRev(docs: any): any; import DatabaseQueryEngine from "./dbInterface"; diff --git a/packages/cozy-pouch-link/types/db/idb/getDocs.d.ts b/packages/cozy-pouch-link/types/db/idb/getDocs.d.ts new file mode 100644 index 0000000000..1d6827e9fb --- /dev/null +++ b/packages/cozy-pouch-link/types/db/idb/getDocs.d.ts @@ -0,0 +1,37 @@ +export const MAX_LIMIT: number; +export function parseResults(client: any, result: any, doctype: any, { isSingleDoc, skip, limit }?: { + isSingleDoc?: boolean; + skip?: number; + limit?: number; +}): { + data: any; + meta?: undefined; + skip?: undefined; + next?: undefined; +} | { + data: any[]; + meta: { + count: number; + }; + skip: number; + next: boolean; +}; +export function queryWithCursor(index: any, idbKeyRange: any, { offset, limit, sortDirection }?: { + offset?: number; + limit?: number; + sortDirection?: string; +}): Promise; +export function queryWithAll(store: any, { limit }?: { + limit?: number; +}): Promise; +export function getAllData(store: any, { limit, skip }?: { + limit?: number; + skip?: number; +}): Promise; +export function findData(store: any, findOpts: any): Promise; +export function getSingleDoc(store: any, id: any): Promise; +export function createIDBIndex(queryEngine: any, { indexName, indexedFields, shouldRecreateIndex }: { + indexName: any; + indexedFields: any; + shouldRecreateIndex?: boolean; +}): Promise; diff --git a/packages/cozy-pouch-link/types/db/idb/idb.d.ts b/packages/cozy-pouch-link/types/db/idb/idb.d.ts new file mode 100644 index 0000000000..02b1a16251 --- /dev/null +++ b/packages/cozy-pouch-link/types/db/idb/idb.d.ts @@ -0,0 +1,9 @@ +export default class IndexedDBQueryEngine extends DatabaseQueryEngine { + constructor(pouchManager: any, doctype: any); + db: IDBDatabase; + client: any; + doctype: any; + storeName: string; + openRequest: IDBOpenDBRequest; +} +import DatabaseQueryEngine from "../dbInterface"; diff --git a/packages/cozy-pouch-link/types/db/idb/mango.d.ts b/packages/cozy-pouch-link/types/db/idb/mango.d.ts new file mode 100644 index 0000000000..16eb6582fa --- /dev/null +++ b/packages/cozy-pouch-link/types/db/idb/mango.d.ts @@ -0,0 +1,16 @@ +export function extractFiltersFromSelector(selector: any, indexedFields: any, { forceInMemory }?: { + forceInMemory?: boolean; +}): { + notForCursorFilters: {}; + cursorFilters: {}; +}; +export function getSortDirection(sort: any): "next" | "prev"; +export function evaluateSelectorInMemory(store: any, data: any, selector: any, sort: any): Promise; +export function executeIDBFind(store: any, { selector, indexedFields, sort, forceInMemory, limit, offset }: { + selector: any; + indexedFields: any; + sort: any; + forceInMemory?: boolean; + limit?: number; + offset?: number; +}): Promise; diff --git a/packages/cozy-pouch-link/types/db/sqlite/sql.d.ts b/packages/cozy-pouch-link/types/db/sqlite/sql.d.ts index 725bf66d90..3a7291f2f1 100644 --- a/packages/cozy-pouch-link/types/db/sqlite/sql.d.ts +++ b/packages/cozy-pouch-link/types/db/sqlite/sql.d.ts @@ -1,4 +1,3 @@ -export function keepDocWitHighestRev(docs: any): any; export function parseResults(client: any, result: any, doctype: any, { isSingleDoc, skip, limit }?: { isSingleDoc?: boolean; skip?: number; diff --git a/packages/cozy-pouch-link/types/errors.d.ts b/packages/cozy-pouch-link/types/errors.d.ts index b845f201ce..ff86fc105f 100644 --- a/packages/cozy-pouch-link/types/errors.d.ts +++ b/packages/cozy-pouch-link/types/errors.d.ts @@ -1,3 +1,4 @@ export function isExpiredTokenError(error: any): any; export function isMissingSQLiteIndexError(error: any): boolean; export function isMissingPouchDBIndexError(error: any): boolean; +export function isMissingIDBIndexError(error: any): boolean; diff --git a/packages/cozy-pouch-link/types/index.d.ts b/packages/cozy-pouch-link/types/index.d.ts index 458f92711b..4e4628fbba 100644 --- a/packages/cozy-pouch-link/types/index.d.ts +++ b/packages/cozy-pouch-link/types/index.d.ts @@ -1,2 +1,3 @@ export { default } from "./CozyPouchLink"; export { default as SQLiteQuery } from "./db/sqlite/sqliteDb"; +export { default as IndexedDBQuery } from "./db/idb/idb";