From 2c87868d7cc023982ee9889cbd1046996fda260e Mon Sep 17 00:00:00 2001 From: Jonathan Brumley Date: Fri, 10 Feb 2023 18:26:54 -0600 Subject: [PATCH 1/7] check if is query object before checking for --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 707aac0..d9bbe7e 100644 --- a/index.js +++ b/index.js @@ -146,7 +146,7 @@ class Collection { const initialDoc = {} for (const queryField of Object.keys(query)) { const queryValue = query[queryField] - if ('$eq' in queryValue) initialDoc[queryField] = queryValue.$eq + if (isQueryObject(queryValue) && '$eq' in queryValue) initialDoc[queryField] = queryValue.$eq else if (!isQueryObject(queryValue)) initialDoc[queryField] = queryValue } From dc234c1b02cefa058d62f20c8feed869a5b7f6ee Mon Sep 17 00:00:00 2001 From: Jonathan Brumley Date: Fri, 10 Feb 2023 18:46:57 -0600 Subject: [PATCH 2/7] upgrade dev dependencies and test --- package.json | 8 ++++---- test.js | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 96d65f9..6f605d7 100644 --- a/package.json +++ b/package.json @@ -29,10 +29,10 @@ "cbor": "^8.1.0" }, "devDependencies": { - "hyperbee": "^1.5.4", - "hypercore": "^9.9.1", - "random-access-memory": "^3.1.2", - "standard": "^16.0.3", + "hyperbee": "^2.0.0", + "hypercore": "^10.7.0", + "random-access-memory": "^6.1.0", + "standard": "^17.0.0", "tape": "^5.2.2" } } diff --git a/test.js b/test.js index 56ae983..6972f11 100644 --- a/test.js +++ b/test.js @@ -6,7 +6,8 @@ const HyperbeeDeeBee = require('./') const { DB } = HyperbeeDeeBee function getBee () { - return new Hyperbee(new Hypercore(RAM)) + const core = new Hypercore(RAM) + return new Hyperbee(core, { extension: false }) } test('Create a document in a collection', async (t) => { From 415d888ab60675981e7ff44d6933544f535d0db8 Mon Sep 17 00:00:00 2001 From: Jonathan Brumley Date: Fri, 10 Feb 2023 18:49:40 -0600 Subject: [PATCH 3/7] upgrade more pkgs --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 6f605d7..49848df 100644 --- a/package.json +++ b/package.json @@ -25,14 +25,14 @@ }, "homepage": "https://github.com/RangerMauve/hyperbeedeebee#readme", "dependencies": { - "bson": "^4.3.0", + "bson": "^5.0.0", "cbor": "^8.1.0" }, "devDependencies": { - "hyperbee": "^2.0.0", + "hyperbee": "^2.4.2", "hypercore": "^10.7.0", "random-access-memory": "^6.1.0", "standard": "^17.0.0", - "tape": "^5.2.2" + "tape": "^5.6.3" } } From 744322857dfd6d1abf8aa070aff890bd92fdeffa Mon Sep 17 00:00:00 2001 From: Jonathan Brumley Date: Fri, 10 Feb 2023 18:51:03 -0600 Subject: [PATCH 4/7] ObjectID => ObjectId --- index.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index d9bbe7e..6bb5f31 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,5 @@ const BSON = require('bson') -const { ObjectID } = BSON +const { ObjectId } = BSON const cbor = require('cbor') // Version of the indexing algorithm @@ -78,7 +78,7 @@ class Collection { if (!doc._id) { doc = { ...doc, - _id: new ObjectID() + _id: new ObjectId() } } @@ -401,7 +401,7 @@ class Cursor { } async * [Symbol.asyncIterator] () { - if (this.query._id && (this.query._id instanceof ObjectID)) { + if (this.query._id && (this.query._id instanceof ObjectId)) { // Doc IDs are unique, so we can query against them without doing a search const key = this.query._id.id @@ -773,8 +773,8 @@ function makeIndexKeyV2 (doc, fields, allFields = fields) { // CBOR encode fields const keyValues = fields.map((field) => { const value = doc[field] - // Detect ObjectID - if (value instanceof ObjectID) { + // Detect ObjectId + if (value instanceof ObjectId) { return value.id } return value @@ -809,7 +809,7 @@ function makeDocFromIndexV2 (key, fields) { const field = fields[index] || '_id' if (Buffer.isBuffer(value) && value.length === 12) { try { - doc[field] = new ObjectID(value) + doc[field] = new ObjectId(value) } catch { doc[field] = value } From e5d4efc2943745ff0847b515e5c307a6c2d41efb Mon Sep 17 00:00:00 2001 From: Jonathan Brumley Date: Sun, 12 Nov 2023 16:36:13 -0600 Subject: [PATCH 5/7] add additional api functions --- index.js | 51 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 6bb5f31..c9c26ca 100644 --- a/index.js +++ b/index.js @@ -41,6 +41,26 @@ const UPDATE_TYPES = { $push: updatePush } +/** + * Parse sort key. + * + * @param {String} key + * @return {Array} + * @type private + */ + +function parseSort (key = '') { + if (Array.isArray(key)) { + const keys = key.map(parseSort) + return Object.assign(...keys) + } + + const [sign] = key.match(/^[+-]/) || [] + if (sign) key = key.substring(1) + const dir = sign === '-' ? -1 : 1 + return [key, dir] +} + class DB { constructor (bee) { this.bee = bee @@ -72,6 +92,18 @@ class Collection { this.idx = bee.sub('idx') } + async insertOne (data) { + const doc = await this.insert(data) + return Object.assign(data, doc) + } + + async insertMany (data) { + for (const item of data) { + await this.insertOne(item) + } + return data + } + async insert (rawDoc) { let doc = rawDoc if (!doc) throw new TypeError('No Document Supplied') @@ -105,6 +137,16 @@ class Collection { return doc } + async updateOne (query = {}, data, options) { + await this.update(query, data, options) + return this.findOne(query) + } + + async updateMany (query = {}, data, options) { + await this.update(query, data, options) + return this.findMany(query) + } + async update (query = {}, update = {}, options = {}) { const { upsert = false, @@ -165,12 +207,13 @@ class Collection { async findOne (query = {}) { const results = await (this.find(query).limit(1)) - const [doc] = results + return doc || null + } - if (!doc) throw new Error('not found') - - return doc + findMany = async (query, { sort = null } = {}) => { + const q = this.find(query) + return sort ? q.sort(...parseSort(sort)) : q } find (query = {}) { From 7657dee210bb3fbfd556ae695866b5f84555c1db Mon Sep 17 00:00:00 2001 From: Jonathan Brumley Date: Sun, 12 Nov 2023 17:28:44 -0600 Subject: [PATCH 6/7] fix findMany --- index.js | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/index.js b/index.js index c9c26ca..4428386 100644 --- a/index.js +++ b/index.js @@ -41,26 +41,6 @@ const UPDATE_TYPES = { $push: updatePush } -/** - * Parse sort key. - * - * @param {String} key - * @return {Array} - * @type private - */ - -function parseSort (key = '') { - if (Array.isArray(key)) { - const keys = key.map(parseSort) - return Object.assign(...keys) - } - - const [sign] = key.match(/^[+-]/) || [] - if (sign) key = key.substring(1) - const dir = sign === '-' ? -1 : 1 - return [key, dir] -} - class DB { constructor (bee) { this.bee = bee @@ -211,7 +191,7 @@ class Collection { return doc || null } - findMany = async (query, { sort = null } = {}) => { + findMany (query, { sort = null } = {}) { const q = this.find(query) return sort ? q.sort(...parseSort(sort)) : q } @@ -574,6 +554,18 @@ class Cursor { } } +function parseSort (key = '') { + if (Array.isArray(key)) { + const keys = key.map(parseSort) + return Object.assign(...keys) + } + + const [sign] = key.match(/^[+-]/) || [] + if (sign) key = key.substring(1) + const dir = sign === '-' ? -1 : 1 + return [key, dir] +} + function performUpdate (doc, update) { if (Array.isArray(update)) { return update.reduce(performUpdate, doc) From 53eaefe9f8cacdc4233f0ad2fb9e095076030bbe Mon Sep 17 00:00:00 2001 From: Jonathan Brumley Date: Mon, 13 Nov 2023 07:34:24 -0600 Subject: [PATCH 7/7] use sub-encoding --- index.js | 69 ++++++++++++++++++++++++++++++++-------------------- package.json | 5 ++-- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/index.js b/index.js index 4428386..717d586 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,7 @@ const BSON = require('bson') const { ObjectId } = BSON const cbor = require('cbor') +const SubEncoder = require('sub-encoder') // Version of the indexing algorithm // Will be incremented for breaking changes @@ -67,11 +68,19 @@ class Collection { constructor (name, bee) { this.name = name this.bee = bee - this.docs = bee.sub('doc') - this.idxs = bee.sub('idxs') - this.idx = bee.sub('idx') + this.enc = new SubEncoder() + + this.idxEncoding = this.enc.sub('idx') + this.idxsEncoding = this.enc.sub('idxs') + this.docsEncoding = this.enc.sub('docs') + + // this.watching() } + // watching () { + // this.watcher = this.bee.watch({ keyEncoding: this.docsEncoding }) + // } + async insertOne (data) { const doc = await this.insert(data) return Object.assign(data, doc) @@ -97,21 +106,21 @@ class Collection { // Get _id as buffer const key = doc._id.id - const exists = await this.docs.get(key) + const exists = await this.bee.get(key, { keyEncoding: this.docsEncoding }) if (exists) throw new Error('Duplicate Key error, try using .update?') const value = BSON.serialize(doc) - await this.docs.put(key, value) + await this.bee.put(key, value, { keyEncoding: this.docsEncoding }) const indexes = await this.listIndexes() for (const { fields, name } of indexes) { // TODO: Cache index subs - const bee = this.idx.sub(name) - await this._indexDocument(bee, fields, doc) + const enc = this.idxEncoding.sub(name) + await this._indexDocument(enc, fields, doc) } return doc @@ -152,14 +161,14 @@ class Collection { const key = doc._id.id const value = BSON.serialize(newDoc) - await this.docs.put(key, value) + await this.bee.put(key, value, { keyEncoding: this.docsEncoding }) for (const { fields, name } of indexes) { // TODO: Cache index subs - const bee = this.idx.sub(name) + const enc = this.idxEncoding.sub(name) - await this._deIndexDocument(bee, fields, doc) - await this._indexDocument(bee, fields, newDoc) + await this._deIndexDocument(enc, fields, doc) + await this._indexDocument(enc, fields, newDoc) } nModified++ } @@ -200,6 +209,11 @@ class Collection { return new Cursor(query, this) } + watch (_id, opts) { + // const key = _id.id + // return this.docsEncoding.watch() + } + async createIndex (fields, { rebuild = false, version = INDEX_VERSION, ...opts } = {}) { const name = fields.join(',') const exists = await this.indexExists(name) @@ -220,7 +234,7 @@ class Collection { opts } - await this.idxs.put(name, BSON.serialize(index)) + await this.bee.put(name, BSON.serialize(index), { keyEncoding: this.idxsEncoding }) await this.reIndex(name) @@ -228,12 +242,12 @@ class Collection { } async indexExists (name) { - const exists = await this.idxs.get(name) + const exists = await this.bee.get(name, { keyEncoding: this.idxsEncoding }) return exists !== null } async getIndex (name) { - const data = await this.idxs.get(name) + const data = await this.bee.get(name, { keyEncoding: this.idxsEncoding }) if (!data) throw new Error('Invalid index') return BSON.deserialize(data.value) } @@ -241,36 +255,36 @@ class Collection { async reIndex (name) { const { fields } = await this.getIndex(name) // TODO: Cache index subs - const bee = this.idx.sub(name) + const enc = this.idxEncoding.sub(name) for await (const doc of this.find()) { - await this._indexDocument(bee, fields, doc) + await this._indexDocument(enc, fields, doc) } } // This is a private API, don't depend on it - async _indexDocument (bee, fields, doc) { + async _indexDocument (enc, fields, doc) { if (!hasFields(doc, fields)) return const idxValue = doc._id.id - const batch = bee.batch() + const batch = this.bee.batch() for (const flattened of flattenDocument(doc, fields)) { const idxKey = makeIndexKeyV2(flattened, fields) - await batch.put(idxKey, idxValue) + await batch.put(idxKey, idxValue, { keyEncoding: enc }) } await batch.flush() } - async _deIndexDocument (bee, fields, doc) { + async _deIndexDocument (enc, fields, doc) { if (!hasFields(doc, fields)) return - const batch = bee.batch() + const batch = this.bee.batch() for (const flattened of flattenDocument(doc, fields)) { const idxKey = makeIndexKeyV2(flattened, fields) - await batch.del(idxKey) + await batch.del(idxKey, { keyEncoding: enc }) } await batch.flush() @@ -278,7 +292,7 @@ class Collection { // TODO: Cache indexes? async listIndexes () { - const stream = this.idxs.createReadStream() + const stream = this.bee.createReadStream({ keyEncoding: this.idxsEncoding }) const indexes = [] for await (const { value } of stream) { @@ -428,7 +442,7 @@ class Cursor { // Doc IDs are unique, so we can query against them without doing a search const key = this.query._id.id - const found = await this.collection.docs.get(key) + const found = await this.collection.bee.get(key, { keyEncoding: this.collection.docsEncoding }) // Exit premaurely @@ -518,7 +532,8 @@ class Cursor { lt[lt.length - 1] = 0xFF } - const stream = this.collection.idx.sub(index.name).createReadStream(opts) + const enc = this.collection.idxEncoding.sub(index.name) + const stream = this.collection.bee.createReadStream({ keyEncoding: enc, ...opts }) for await (const { key, value: rawId } of stream) { const keyDoc = makeDocFromIndex(key, index.fields) @@ -526,7 +541,7 @@ class Cursor { // Test the fields agains the index to avoid fetching the doc if (!matchesQuery(keyDoc, subQuery)) continue - const { value: rawDoc } = await this.collection.docs.get(rawId) + const { value: rawDoc } = await this.collection.bee.get(rawId, { keyEncoding: this.collection.docsEncoding }) const doc = BSON.deserialize(rawDoc) // TODO: Avoid needing to double-process the values @@ -537,7 +552,7 @@ class Cursor { } } else if (sort === null) { // If we aren't sorting, and don't have an index, iterate over all docs - const stream = this.collection.docs.createReadStream() + const stream = this.collection.bee.createReadStream({ keyEncoding: this.collection.docsEncoding }) for await (const { value: rawDoc } of stream) { // TODO: Can we avoid iterating over keys that should be skipped? diff --git a/package.json b/package.json index 49848df..a9eeb05 100644 --- a/package.json +++ b/package.json @@ -26,10 +26,11 @@ "homepage": "https://github.com/RangerMauve/hyperbeedeebee#readme", "dependencies": { "bson": "^5.0.0", - "cbor": "^8.1.0" + "cbor": "^8.1.0", + "sub-encoder": "^2.1.3" }, "devDependencies": { - "hyperbee": "^2.4.2", + "hyperbee": "^2.16.0", "hypercore": "^10.7.0", "random-access-memory": "^6.1.0", "standard": "^17.0.0",