From 82526e75f0e36db44a1f66d9bdc5213b26844fb1 Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Mon, 16 Mar 2026 17:21:19 -0700 Subject: [PATCH 01/21] wip: basic working authenticated PHT node put/get --- examples/keyword-search/package.json | 6 ++ examples/keyword-search/server.js | 55 ++++++++++++++++++ index.js | 84 ++++++++++++++++++++++++++++ lib/constants.js | 19 +++++-- lib/messages.js | 55 ++++++++++++++++++ lib/persistent.js | 33 +++++++++++ 6 files changed, 248 insertions(+), 4 deletions(-) create mode 100644 examples/keyword-search/package.json create mode 100644 examples/keyword-search/server.js diff --git a/examples/keyword-search/package.json b/examples/keyword-search/package.json new file mode 100644 index 00000000..0c5dc6ca --- /dev/null +++ b/examples/keyword-search/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "b4a": "^1.8.0", + "hyperdht": "^6.29.2" + } +} diff --git a/examples/keyword-search/server.js b/examples/keyword-search/server.js new file mode 100644 index 00000000..5d9ec160 --- /dev/null +++ b/examples/keyword-search/server.js @@ -0,0 +1,55 @@ +const b4a = require('b4a') +const { PrefixHashTree } = require('prefix-hash-tree') +const createTestnet = require('../../testnet.js') + +class AsyncMap { + constructor() { + this.map = new Map() + } + + async get(key) { + return new Promise((resolve) => { + resolve(this.map.get(key.toString('hex')) || null) + }) + } + + async put(key, val) { + return new Promise((resolve) => { + this.map.set(key.toString('hex'), val) + resolve(val) + }) + } +} + +async function main() { + const testnet = await createTestnet( + 10, + { + port: 49739 + } + ) + + const precompute = new AsyncMap() + + const localPHT = new PrefixHashTree({ + bitDomain: 320, + getFunc: precompute.get.bind(precompute), + putFunc: precompute.put.bind(precompute) + }) + + await localPHT.init() + + const vocab = ['dog', 'cat', 'bird', 'fish', 'tree', 'human', 'car', 'boat', 'plane', 'train'] + + for (const keyword of vocab) { + await localPHT.insert(localPHT.unhashedKeyFrom(keyword), null) + } + + const topologyID = b4a.from('myCoolTopology', 'utf8') + + for (const [_, phtNode] of precompute.map) { + await testnet.nodes[0].authenticatedPHTNodePut(testnet.nodes[0].defaultKeyPair, topologyID, phtNode) + } +} + +main() \ No newline at end of file diff --git a/index.js b/index.js index d9681567..c1362628 100644 --- a/index.js +++ b/index.js @@ -16,6 +16,8 @@ const { decode } = require('hypercore-id-encoding') const RawStreamSet = require('./lib/raw-stream-set') const ConnectionPool = require('./lib/connection-pool') const { STREAM_NOT_CONNECTED } = require('./lib/errors') +const { PrefixHashTree } = require('prefix-hash-tree') +const { label } = require('prefix-hash-tree/node') const DEFAULTS = { ...DHT.DEFAULTS, @@ -389,6 +391,63 @@ class HyperDHT extends DHT { return { publicKey: keyPair.publicKey, closestNodes: query.closestNodes, seq, signature } } + async authenticatedPHTNodeGet(target, opts = {}) { + opts = { ...opts, map: mapAuthenticatedPHTNode } + const query = this.query({ target, command: COMMANDS.AUTHENTICATED_PHT_NODE_GET, value: null }, opts) + + for await (const node of query) { + const { phtNode } = node + + // TODO: implement Persistent.verifyAuthenticatedPHTNode and verify here + // (what else must we verify?) + + return node + } + + return null + } + + async authenticatedPHTNodePut(keyPair, topologyID, phtNode, opts = {}) { + const signAuthenticatedPHTNode = + opts.signAuthenticatedPHTNode || Persistent.signAuthenticatedPHTNode + + const indexID = b4a.allocUnsafe(32) + sodium.crypto_generichash(indexID, b4a.concat([keyPair.publicKey, topologyID])) + const target = b4a.allocUnsafe(32) + + // TODO: this re-implements PrefixHashTree._labelHash because we're putting nodes 'a la carte' + // outside the context of a PrefixHashTree instance. Is there a better way? + sodium.crypto_generichash(target, b4a.from(`${indexID.toString('hex')}${label(phtNode)}`)) + + const signature = await signAuthenticatedPHTNode(phtNode, topologyID, keyPair) + + const signed = c.encode(m.authenticatedPHTNodePutRequest, { + publicKey: keyPair.publicKey, + topologyID: topologyID, + phtNode: phtNode, + signature: signature + }) + + opts = { + ...opts, + map: mapAuthenticatedPHTNode, + commit(reply, dht) { + return dht.request( + { token: reply.token, target, command: COMMANDS.AUTHENTICATED_PHT_NODE_PUT, value: signed }, + reply.from + ) + } + } + + const query = this.query( + { target, command: COMMANDS.AUTHENTICATED_PHT_NODE_GET, value: null }, + opts + ) + await query.finished() + + return { indexID: indexID, closestNodes: query.closestNodes } + } + onrequest(req) { switch (req.command) { case COMMANDS.PEER_HANDSHAKE: { @@ -436,6 +495,14 @@ class HyperDHT extends DHT { this._persistent.onimmutableget(req) return true } + case COMMANDS.AUTHENTICATED_PHT_NODE_PUT: { + this._persistent.onauthenticatedphtnodeput(req) + return true + } + case COMMANDS.AUTHENTICATED_PHT_NODE_GET: { + this._persistent.onauthenticatedphtnodeget(req) + return true + } } return false @@ -580,6 +647,23 @@ function mapMutable(node) { } } +function mapAuthenticatedPHTNode(node) { + if (!node.value) return null + + try { + const { node: phtNode } = c.decode(m.authenticatedPHTNodeGetResponse, node.value) + + return { + token: node.token, + from: node.from, + to: node.to, + node: phtNode + } + } catch { + return null + } +} + function noop() {} function filterNode(node) { diff --git a/lib/constants.js b/lib/constants.js index 0d43cd5e..fb386090 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -10,7 +10,9 @@ const COMMANDS = (exports.COMMANDS = { MUTABLE_PUT: 6, MUTABLE_GET: 7, IMMUTABLE_PUT: 8, - IMMUTABLE_GET: 9 + IMMUTABLE_GET: 9, + AUTHENTICATED_PHT_NODE_PUT: 10, + AUTHENTICATED_PHT_NODE_GET: 11 }) exports.BOOTSTRAP_NODES = global.Pear?.config.dht?.bootstrap || [ @@ -39,13 +41,21 @@ exports.ERROR = { SEQ_TOO_LOW: 17 } -const [NS_ANNOUNCE, NS_UNANNOUNCE, NS_MUTABLE_PUT, NS_PEER_HANDSHAKE, NS_PEER_HOLEPUNCH] = +const [ + NS_ANNOUNCE, + NS_UNANNOUNCE, + NS_MUTABLE_PUT, + NS_PEER_HANDSHAKE, + NS_PEER_HOLEPUNCH, + NS_AUTHENTICATED_PHT_NODE_PUT +] = crypto.namespace('hyperswarm/dht', [ COMMANDS.ANNOUNCE, COMMANDS.UNANNOUNCE, COMMANDS.MUTABLE_PUT, COMMANDS.PEER_HANDSHAKE, - COMMANDS.PEER_HOLEPUNCH + COMMANDS.PEER_HOLEPUNCH, + COMMANDS.AUTHENTICATED_PHT_NODE_PUT ]) exports.NS = { @@ -53,5 +63,6 @@ exports.NS = { UNANNOUNCE: NS_UNANNOUNCE, MUTABLE_PUT: NS_MUTABLE_PUT, PEER_HANDSHAKE: NS_PEER_HANDSHAKE, - PEER_HOLEPUNCH: NS_PEER_HOLEPUNCH + PEER_HOLEPUNCH: NS_PEER_HOLEPUNCH, + AUTHENTICATED_PHT_NODE_PUT: NS_AUTHENTICATED_PHT_NODE_PUT } diff --git a/lib/messages.js b/lib/messages.js index b7fb4612..3db8ca58 100644 --- a/lib/messages.js +++ b/lib/messages.js @@ -420,3 +420,58 @@ exports.mutableGetResponse = { } } } + +exports.authenticatedPHTNodeSignable = { + preencode(state, m) { + c.buffer.preencode(state, m.topologyID) + c.json.preencode(state, m.phtNode) + }, + encode(state, m) { + c.buffer.encode(state, m.topologyID) + c.json.encode(state, m.phtNode) + }, + decode(state, m) { + return { + topologyID: c.buffer.decode(state), + node: c.json.decode(state) + } + } +} + +exports.authenticatedPHTNodePutRequest = { + preencode(state, m) { + c.fixed32.preencode(state, m.publicKey) + c.buffer.preencode(state, m.topologyID) + c.json.preencode(state, m.phtNode) + c.fixed64.preencode(state, m.signature) + }, + encode(state, m) { + c.fixed32.encode(state, m.publicKey) + c.buffer.encode(state, m.topologyID) + c.json.encode(state, m.phtNode) + c.fixed64.encode(state, m.signature) + }, + decode(state) { + return { + publicKey: c.fixed32.decode(state), + topologyID: c.buffer.decode(state), + phtNode: c.json.decode(state), + signature: c.fixed64.decode(state) + } + } +} + +// TODO: send publicKey, topologyID, and signature for recipient verification +exports.authenticatedPHTNodeGetResponse = { + preencode(state, m) { + c.json.preencode(state, m.phtNode) + }, + encode(state, m) { + c.json.encode(state, m.phtNode) + }, + decode(state) { + return { + phtNode: c.json.decode(state) + } + } +} \ No newline at end of file diff --git a/lib/persistent.js b/lib/persistent.js index a97fea23..4689e9cb 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -21,6 +21,7 @@ module.exports = class Persistent { this.refreshes = new Cache(opts.refreshes) this.mutables = new Cache(opts.mutables) this.immutables = new Cache(opts.immutables) + this.authenticatedPHTNodes = new Cache(opts.records) } onlookup(req) { @@ -226,11 +227,33 @@ module.exports = class Persistent { req.reply(null) } + // TODO: does this require any additional sophistication? + onauthenticatedphtnodeget(req) { + if (!req.target) return + + const k = b4a.toString(req.target, 'hex') + const phtNode = this.authenticatedPHTNodes.get(k) + + req.reply(phtNode || null) + } + + onauthenticatedphtnodeput(req) { + if (!req.target || !req.token || !req.value) return + + // TODO: authentication! + + const k = b4a.toString(req.target, 'hex') + this.authenticatedPHTNodes.set(k, unslab(req.value)) + + req.reply(null) + } + destroy() { this.records.destroy() this.refreshes.destroy() this.mutables.destroy() this.immutables.destroy() + this.authenticatedPHTNodes.destroy() } static signMutable(seq, value, keyPair) { @@ -254,6 +277,16 @@ module.exports = class Persistent { static signUnannounce(target, token, id, ann, keyPair) { return sign(annSignable(target, token, id, ann, NS.UNANNOUNCE), keyPair) } + + static signAuthenticatedPHTNode(phtNode, topologyID, keyPair) { + const signable = b4a.allocUnsafe(32 + 32) + const hash = signable.subarray(32) + + signable.set(NS.AUTHENTICATED_PHT_NODE_PUT, 0) + + sodium.crypto_generichash(hash, c.encode(m.authenticatedPHTNodeSignable, { topologyID, phtNode })) + return sign(signable, keyPair) + } } function verifyMutable(signature, seq, value, publicKey) { From 10b99ffaaaecc7222c864fe096b810bb68b49347 Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Tue, 17 Mar 2026 15:38:39 -0700 Subject: [PATCH 02/21] make PHT node get decoding work, demo it in keyword-search example --- examples/keyword-search/index.js | 74 ++++++++++++++++++++++++++++ examples/keyword-search/package.json | 3 +- examples/keyword-search/server.js | 55 --------------------- index.js | 8 +-- lib/messages.js | 12 ++++- lib/persistent.js | 1 + 6 files changed, 90 insertions(+), 63 deletions(-) create mode 100644 examples/keyword-search/index.js delete mode 100644 examples/keyword-search/server.js diff --git a/examples/keyword-search/index.js b/examples/keyword-search/index.js new file mode 100644 index 00000000..a2293b1b --- /dev/null +++ b/examples/keyword-search/index.js @@ -0,0 +1,74 @@ +const b4a = require('b4a') +const { PrefixHashTree } = require('prefix-hash-tree') +const createTestnet = require('../../testnet.js') + +class AsyncMap { + constructor() { + this.map = new Map() + } + + async get(key) { + return new Promise((resolve) => { + resolve(this.map.get(key.toString('hex')) || null) + }) + } + + async put(key, val) { + return new Promise((resolve) => { + this.map.set(key.toString('hex'), val) + resolve(val) + }) + } +} + +async function main() { + const testnet = await createTestnet( + 30, + { + port: 49739 + } + ) + + // Precompute a static trie topology using a local datastore + const localNodeTable = new AsyncMap() + + const localPHT = new PrefixHashTree({ + bitDomain: 320, + getFunc: localNodeTable.get.bind(localNodeTable), + putFunc: localNodeTable.put.bind(localNodeTable) + }) + + await localPHT.init() + + const coolVocab = ['dog', 'cat', 'bird', 'fish', 'tree', 'human', 'car', 'boat', 'plane', 'train'] + + for (const keyword of coolVocab) { + await localPHT.insert(localPHT.unhashedKeyFrom(keyword), [`value for ${keyword}`]) + } + + const topologyID = b4a.from('coolVocab', 'utf8') + + // Store the precomputed topology the DHT, note the indexID + let indexID + for (const [_, phtNode] of localNodeTable.map) { + const { target, indexID: id, _ } = + await testnet.nodes[0].authenticatedPHTNodePut(testnet.nodes[0].defaultKeyPair, topologyID, phtNode) + indexID = id + } + + // Now anyone can use that indexID to search the DHT + const searchIndex = new PrefixHashTree({ + indexID: indexID.toString('hex'), + bitDomain: 320, + getFunc: testnet.nodes[0].authenticatedPHTNodeGet.bind(testnet.nodes[0]), + putFunc: null + }) + + const query = await searchIndex.prefixQuery(searchIndex.prefixFrom('ca')) + console.log(`Discovered keywords ${query.map(([key, _]) => key)}`) + + const search = await searchIndex.searchExact(searchIndex.unhashedKeyFrom('car')) + console.log(search) +} + +main() \ No newline at end of file diff --git a/examples/keyword-search/package.json b/examples/keyword-search/package.json index 0c5dc6ca..1381bbdb 100644 --- a/examples/keyword-search/package.json +++ b/examples/keyword-search/package.json @@ -1,6 +1,5 @@ { "dependencies": { - "b4a": "^1.8.0", - "hyperdht": "^6.29.2" + "b4a": "^1.8.0" } } diff --git a/examples/keyword-search/server.js b/examples/keyword-search/server.js deleted file mode 100644 index 5d9ec160..00000000 --- a/examples/keyword-search/server.js +++ /dev/null @@ -1,55 +0,0 @@ -const b4a = require('b4a') -const { PrefixHashTree } = require('prefix-hash-tree') -const createTestnet = require('../../testnet.js') - -class AsyncMap { - constructor() { - this.map = new Map() - } - - async get(key) { - return new Promise((resolve) => { - resolve(this.map.get(key.toString('hex')) || null) - }) - } - - async put(key, val) { - return new Promise((resolve) => { - this.map.set(key.toString('hex'), val) - resolve(val) - }) - } -} - -async function main() { - const testnet = await createTestnet( - 10, - { - port: 49739 - } - ) - - const precompute = new AsyncMap() - - const localPHT = new PrefixHashTree({ - bitDomain: 320, - getFunc: precompute.get.bind(precompute), - putFunc: precompute.put.bind(precompute) - }) - - await localPHT.init() - - const vocab = ['dog', 'cat', 'bird', 'fish', 'tree', 'human', 'car', 'boat', 'plane', 'train'] - - for (const keyword of vocab) { - await localPHT.insert(localPHT.unhashedKeyFrom(keyword), null) - } - - const topologyID = b4a.from('myCoolTopology', 'utf8') - - for (const [_, phtNode] of precompute.map) { - await testnet.nodes[0].authenticatedPHTNodePut(testnet.nodes[0].defaultKeyPair, topologyID, phtNode) - } -} - -main() \ No newline at end of file diff --git a/index.js b/index.js index c1362628..b49a996b 100644 --- a/index.js +++ b/index.js @@ -401,7 +401,7 @@ class HyperDHT extends DHT { // TODO: implement Persistent.verifyAuthenticatedPHTNode and verify here // (what else must we verify?) - return node + return phtNode } return null @@ -445,7 +445,7 @@ class HyperDHT extends DHT { ) await query.finished() - return { indexID: indexID, closestNodes: query.closestNodes } + return { target: target, indexID: indexID, closestNodes: query.closestNodes } } onrequest(req) { @@ -651,13 +651,13 @@ function mapAuthenticatedPHTNode(node) { if (!node.value) return null try { - const { node: phtNode } = c.decode(m.authenticatedPHTNodeGetResponse, node.value) + const { phtNode } = c.decode(m.authenticatedPHTNodeGetResponse, node.value) return { token: node.token, from: node.from, to: node.to, - node: phtNode + phtNode: phtNode } } catch { return null diff --git a/lib/messages.js b/lib/messages.js index 3db8ca58..a60bbd7b 100644 --- a/lib/messages.js +++ b/lib/messages.js @@ -461,17 +461,25 @@ exports.authenticatedPHTNodePutRequest = { } } -// TODO: send publicKey, topologyID, and signature for recipient verification exports.authenticatedPHTNodeGetResponse = { preencode(state, m) { + c.fixed32.preencode(state, m.publicKey) + c.buffer.preencode(state, m.topologyID) c.json.preencode(state, m.phtNode) + c.fixed64.preencode(state, m.signature) }, encode(state, m) { + c.fixed32.encode(state, m.publicKey) + c.buffer.encode(state, m.topologyID) c.json.encode(state, m.phtNode) + c.fixed64.encode(state, m.signature) }, decode(state) { return { - phtNode: c.json.decode(state) + publicKey: c.fixed32.decode(state), + topologyID: c.buffer.decode(state), + phtNode: c.json.decode(state), + signature: c.fixed64.decode(state) } } } \ No newline at end of file diff --git a/lib/persistent.js b/lib/persistent.js index 15c03feb..17c0648c 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -243,6 +243,7 @@ module.exports = class Persistent { // TODO: authentication! + // TODO: must we store the whole authenticatedPHTNodePutRequest, or can we discard some of it? const k = b4a.toString(req.target, 'hex') this.authenticatedPHTNodes.set(k, unslab(req.value)) From 541c30a5ddac15409e29b158ae2a3e2373b9324f Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Wed, 18 Mar 2026 15:26:27 -0700 Subject: [PATCH 03/21] add MVP shard replication and e2e example --- examples/keyword-search/index.js | 131 ++++++++++++++++++++++--------- index.js | 57 ++++++++++++++ lib/constants.js | 4 +- lib/messages.js | 31 ++++++++ lib/persistent.js | 31 +++++++- 5 files changed, 215 insertions(+), 39 deletions(-) diff --git a/examples/keyword-search/index.js b/examples/keyword-search/index.js index a2293b1b..885ebb3c 100644 --- a/examples/keyword-search/index.js +++ b/examples/keyword-search/index.js @@ -1,39 +1,21 @@ const b4a = require('b4a') const { PrefixHashTree } = require('prefix-hash-tree') +const { label } = require('prefix-hash-tree/node') const createTestnet = require('../../testnet.js') -class AsyncMap { - constructor() { - this.map = new Map() - } +async function main() { + const testnet = await createTestnet(20, { port: 49739 }) - async get(key) { - return new Promise((resolve) => { - resolve(this.map.get(key.toString('hex')) || null) - }) - } + // Introducing: Keyword Search (featuring Bob and Alice) - async put(key, val) { - return new Promise((resolve) => { - this.map.set(key.toString('hex'), val) - resolve(val) - }) - } -} + const bob = testnet.nodes[0] + const alice = testnet.nodes[1] -async function main() { - const testnet = await createTestnet( - 30, - { - port: 49739 - } - ) - - // Precompute a static trie topology using a local datastore + // Bob precomputes a static trie topology over a finite vocabulary using a local datastore const localNodeTable = new AsyncMap() const localPHT = new PrefixHashTree({ - bitDomain: 320, + bitDomain: 80, getFunc: localNodeTable.get.bind(localNodeTable), putFunc: localNodeTable.put.bind(localNodeTable) }) @@ -43,32 +25,109 @@ async function main() { const coolVocab = ['dog', 'cat', 'bird', 'fish', 'tree', 'human', 'car', 'boat', 'plane', 'train'] for (const keyword of coolVocab) { - await localPHT.insert(localPHT.unhashedKeyFrom(keyword), [`value for ${keyword}`]) + await localPHT.insert(localPHT.unhashedKeyFrom(keyword), `shard pointer for ${keyword}`) } const topologyID = b4a.from('coolVocab', 'utf8') - // Store the precomputed topology the DHT, note the indexID + // Bob stores the precomputed trie topology to the DHT, noting the indexID let indexID for (const [_, phtNode] of localNodeTable.map) { const { target, indexID: id, _ } = - await testnet.nodes[0].authenticatedPHTNodePut(testnet.nodes[0].defaultKeyPair, topologyID, phtNode) + await bob.authenticatedPHTNodePut(bob.defaultKeyPair, topologyID, phtNode) indexID = id } - // Now anyone can use that indexID to search the DHT + console.log(`Bob created a keyword search index (ID ${indexID.toString('hex')})`) + + // (Bob shares the indexID with Alice...) + + // Now Alice can use that indexID to discover valid keywords const searchIndex = new PrefixHashTree({ indexID: indexID.toString('hex'), - bitDomain: 320, - getFunc: testnet.nodes[0].authenticatedPHTNodeGet.bind(testnet.nodes[0]), + bitDomain: 80, + getFunc: alice.authenticatedPHTNodeGet.bind(alice), putFunc: null }) - const query = await searchIndex.prefixQuery(searchIndex.prefixFrom('ca')) - console.log(`Discovered keywords ${query.map(([key, _]) => key)}`) + const query1 = await searchIndex.prefixQuery(searchIndex.prefixFrom('ca')) + console.log(`Alice discovered keywords ${query1.map(([key, _]) => key).join(', ')}`) + + const query2 = await searchIndex.prefixQuery(searchIndex.prefixFrom('t')) + console.log(`Alice discovered keywords ${query2.map(([key, _]) => key).join(', ')}`) + + // And Alice can add her document records under any valid keyword + const catOnATrainRecord = b4a.from('cat_on_a_train.gif', 'utf8') + const catOnAPlaneRecord = b4a.from('cat_on_a_plane.gif', 'utf8') + + const catKey = searchIndex.unhashedKeyFrom('cat') + const catShard = await searchIndex.searchLeaf(catKey) + const catTarget = searchIndex._labelHash(label(catShard)) + + await alice.phtShardPut(catTarget, catKey, catOnATrainRecord) + console.log(`Alice inserted ${catOnATrainRecord} under ${catKey}`) + + await alice.phtShardPut(catTarget, catKey, catOnAPlaneRecord) + console.log(`Alice inserted ${catOnAPlaneRecord} under ${catKey}`) + + const trainKey = searchIndex.unhashedKeyFrom('train') + const trainShard = await searchIndex.searchLeaf(trainKey) + const trainTarget = searchIndex._labelHash(label(trainShard)) + + await alice.phtShardPut(trainTarget, trainKey, catOnATrainRecord) + console.log(`Alice inserted ${catOnATrainRecord} under ${trainKey}`) + + const planeKey = searchIndex.unhashedKeyFrom('plane') + const planeShard = await searchIndex.searchLeaf(planeKey) + const planeTarget = searchIndex._labelHash(label(planeShard)) + + await alice.phtShardPut(planeTarget, planeKey, catOnAPlaneRecord) + console.log(`Alice inserted ${catOnAPlaneRecord} under ${planeKey}`) - const search = await searchIndex.searchExact(searchIndex.unhashedKeyFrom('car')) - console.log(search) + // Later, Bob performs a search for "cat AND train" + const bobSearchIndex = new PrefixHashTree({ + indexID: indexID.toString('hex'), + bitDomain: 80, + getFunc: bob.authenticatedPHTNodeGet.bind(bob), + putFunc: null + }) + + const bobCatKey = bobSearchIndex.unhashedKeyFrom('cat') + const bobCatShard = await bobSearchIndex.searchLeaf(bobCatKey) + const bobCatTarget = bobSearchIndex._labelHash(label(bobCatShard)) + + const foundCatDocs = + (await bob.phtShardGet(bobCatTarget, bobCatKey)).value.records.map(record => b4a.from(record).toString('utf8')) + + const bobPlaneKey = bobSearchIndex.unhashedKeyFrom('plane') + const bobPlaneShard = await bobSearchIndex.searchLeaf(bobPlaneKey) + const bobPlaneTarget = bobSearchIndex._labelHash(label(bobPlaneShard)) + + const foundPlaneDocs = + (await bob.phtShardGet(bobPlaneTarget, bobPlaneKey)).value.records.map(record => b4a.from(record).toString('utf8')) + + const intersection = [...new Set(foundCatDocs.filter(x => foundPlaneDocs.includes(x)))] + + console.log(`Bob searched '${bobCatKey} AND ${bobPlaneKey}' and found: ${intersection}`) +} + +class AsyncMap { + constructor() { + this.map = new Map() + } + + async get(key) { + return new Promise((resolve) => { + resolve(this.map.get(key.toString('hex')) || null) + }) + } + + async put(key, val) { + return new Promise((resolve) => { + this.map.set(key.toString('hex'), val) + resolve(val) + }) + } } main() \ No newline at end of file diff --git a/index.js b/index.js index b49a996b..935745c1 100644 --- a/index.js +++ b/index.js @@ -448,6 +448,44 @@ class HyperDHT extends DHT { return { target: target, indexID: indexID, closestNodes: query.closestNodes } } + async phtShardGet(target, key, opts = {}) { + opts = { ...opts, map: mapPHTShard } + const query = this.query({ target, command: COMMANDS.PHT_SHARD_GET, value: key }, opts) + + for await (const node of query) { + const { value } = node + + // TODO: merge the responses + + return node + } + + return null + } + + async phtShardPut(target, key, value, opts = {}) { + const encoded = c.encode(m.phtShardPutRequest, { + key: key, + value: value + }) + + opts = { + ...opts, + map: mapPHTShard, + commit(reply, dht) { + return dht.request( + { token: reply.token, target, command: COMMANDS.PHT_SHARD_PUT, value: encoded }, + reply.from + ) + } + } + + const query = this.query({ target, command: COMMANDS.PHT_SHARD_GET, value: key }, opts) + await query.finished() + + return { closestNodes: query.closestNodes } + } + onrequest(req) { switch (req.command) { case COMMANDS.PEER_HANDSHAKE: { @@ -503,6 +541,14 @@ class HyperDHT extends DHT { this._persistent.onauthenticatedphtnodeget(req) return true } + case COMMANDS.PHT_SHARD_PUT: { + this._persistent.onphtshardput(req) + return true + } + case COMMANDS.PHT_SHARD_GET: { + this._persistent.onphtshardget(req) + return true + } } return false @@ -664,6 +710,17 @@ function mapAuthenticatedPHTNode(node) { } } +function mapPHTShard(node) { + if (!node.value) return null + + return { + token: node.token, + from: node.from, + to: node.to, + value: c.decode(m.phtShardGetResponse, node.value) + } +} + function noop() {} function filterNode(node) { diff --git a/lib/constants.js b/lib/constants.js index fb386090..06e8ae0a 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -12,7 +12,9 @@ const COMMANDS = (exports.COMMANDS = { IMMUTABLE_PUT: 8, IMMUTABLE_GET: 9, AUTHENTICATED_PHT_NODE_PUT: 10, - AUTHENTICATED_PHT_NODE_GET: 11 + AUTHENTICATED_PHT_NODE_GET: 11, + PHT_SHARD_PUT: 12, + PHT_SHARD_GET: 13 }) exports.BOOTSTRAP_NODES = global.Pear?.config.dht?.bootstrap || [ diff --git a/lib/messages.js b/lib/messages.js index a60bbd7b..091bbfd7 100644 --- a/lib/messages.js +++ b/lib/messages.js @@ -482,4 +482,35 @@ exports.authenticatedPHTNodeGetResponse = { signature: c.fixed64.decode(state) } } +} + +exports.phtShardPutRequest = { + preencode(state, m) { + c.buffer.preencode(state, m.key) + c.buffer.preencode(state, m.value) + }, + encode(state, m) { + c.buffer.encode(state, m.key) + c.buffer.encode(state, m.value) + }, + decode(state) { + return { + key: c.buffer.decode(state), + value: c.buffer.decode(state) + } + } +} + +exports.phtShardGetResponse = { + preencode(state, m) { + c.json.preencode(state, m.records) + }, + encode(state, m) { + c.json.encode(state, m.records) + }, + decode(state) { + return { + records: c.json.decode(state) + } + } } \ No newline at end of file diff --git a/lib/persistent.js b/lib/persistent.js index 17c0648c..d8c4c2b9 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -22,6 +22,7 @@ module.exports = class Persistent { this.mutables = new Cache(opts.mutables) this.immutables = new Cache(opts.immutables) this.authenticatedPHTNodes = new Cache(opts.records) + this.phtShards = new RecordCache(opts.records) } onlookup(req) { @@ -233,9 +234,9 @@ module.exports = class Persistent { if (!req.target) return const k = b4a.toString(req.target, 'hex') - const phtNode = this.authenticatedPHTNodes.get(k) + const phtGetResponse = this.authenticatedPHTNodes.get(k) - req.reply(phtNode || null) + req.reply(phtGetResponse || null) } onauthenticatedphtnodeput(req) { @@ -250,6 +251,32 @@ module.exports = class Persistent { req.reply(null) } + onphtshardget(req) { + if (!req.target) return + + // TODO: get and decode the stored PHT node and validate the PHT key against it + + const k = b4a.concat([req.target, req.value]).toString('hex') + + // TODO: parameterize get qty? + const docs = this.phtShards.get(k, 1000) + const encoded = c.encode(m.phtShardGetResponse, { records: docs }) + + req.reply(encoded) + } + + onphtshardput(req) { + if (!req.target || !req.token || !req.value) return + + // TODO: get and decode the stored PHT node and validate the PHT key against it + + const { key, value } = c.decode(m.phtShardPutRequest, unslab(req.value)) + const k = b4a.concat([req.target, key]).toString('hex') + this.phtShards.add(k, value) + + req.reply(null) + } + destroy() { this.records.destroy() this.refreshes.destroy() From 46551705388a3dc16dade4542bfb134eb46cd3fc Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Fri, 20 Mar 2026 12:15:35 -0700 Subject: [PATCH 04/21] API refactor: `authenticatedPHTNodeGet` returns the hyperdht node object, defer extraction of the wrapped PHT node to the PHT --- examples/keyword-search/index.js | 10 ++++++++-- index.js | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/examples/keyword-search/index.js b/examples/keyword-search/index.js index 885ebb3c..e63b3ec7 100644 --- a/examples/keyword-search/index.js +++ b/examples/keyword-search/index.js @@ -46,7 +46,10 @@ async function main() { const searchIndex = new PrefixHashTree({ indexID: indexID.toString('hex'), bitDomain: 80, - getFunc: alice.authenticatedPHTNodeGet.bind(alice), + getFunc: async (target) => { + const res = await alice.authenticatedPHTNodeGet(target) + return res === null ? res : res.phtNode + }, putFunc: null }) @@ -88,7 +91,10 @@ async function main() { const bobSearchIndex = new PrefixHashTree({ indexID: indexID.toString('hex'), bitDomain: 80, - getFunc: bob.authenticatedPHTNodeGet.bind(bob), + getFunc: async (target) => { + const res = await bob.authenticatedPHTNodeGet(target) + return res === null ? res : res.phtNode + }, putFunc: null }) diff --git a/index.js b/index.js index 22e63995..645948c6 100644 --- a/index.js +++ b/index.js @@ -401,7 +401,7 @@ class HyperDHT extends DHT { // TODO: implement Persistent.verifyAuthenticatedPHTNode and verify here // (what else must we verify?) - return phtNode + return node } return null From 4e73fdb97a7f2a398f19498bdae293c0d2c2d96d Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Mon, 23 Mar 2026 15:47:55 -0700 Subject: [PATCH 05/21] MVP-grade shard storage and replication: announce shard on PHT node put --- lib/persistent.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/persistent.js b/lib/persistent.js index d8c4c2b9..c1efd376 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -239,7 +239,7 @@ module.exports = class Persistent { req.reply(phtGetResponse || null) } - onauthenticatedphtnodeput(req) { + async onauthenticatedphtnodeput(req) { if (!req.target || !req.token || !req.value) return // TODO: authentication! @@ -247,7 +247,12 @@ module.exports = class Persistent { // TODO: must we store the whole authenticatedPHTNodePutRequest, or can we discard some of it? const k = b4a.toString(req.target, 'hex') this.authenticatedPHTNodes.set(k, unslab(req.value)) - + + // TODO: this is the wrong layer of abstraction to do this, because we're announcing shard + // replication but the server is implemented at the hypersearch layer... + const stream = this.dht.announce(req.target, this.dht.defaultKeyPair) + await stream.finished() + req.reply(null) } From fb69df0459e1144a01faf44dbf21d3e8e0ba55f3 Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Thu, 26 Mar 2026 11:53:25 -0700 Subject: [PATCH 06/21] improved shard replication announcement behavior --- lib/persistent.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/persistent.js b/lib/persistent.js index c1efd376..ef57d6d6 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -4,6 +4,7 @@ const RecordCache = require('record-cache') const Cache = require('xache') const b4a = require('b4a') const unslab = require('unslab') +const { isLeaf } = require('prefix-hash-tree/node') const { encodeUnslab } = require('./encode') const m = require('./messages') @@ -244,15 +245,23 @@ module.exports = class Persistent { // TODO: authentication! + const unslabbed = unslab(req.value) + const p = decode(m.authenticatedPHTNodePutRequest, unslabbed) + if (p === null) return + + const { _publicKey, _topologyID, phtNode, _signature } = p + // TODO: must we store the whole authenticatedPHTNodePutRequest, or can we discard some of it? const k = b4a.toString(req.target, 'hex') - this.authenticatedPHTNodes.set(k, unslab(req.value)) + this.authenticatedPHTNodes.set(k, unslabbed) // TODO: this is the wrong layer of abstraction to do this, because we're announcing shard // replication but the server is implemented at the hypersearch layer... - const stream = this.dht.announce(req.target, this.dht.defaultKeyPair) - await stream.finished() - + if (isLeaf(phtNode)) { + const stream = this.dht.announce(req.target, this.dht.defaultKeyPair) + await stream.finished() + } + req.reply(null) } From 7e97b3ad49acf02bc44af2b600060790bba233b3 Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Mon, 30 Mar 2026 14:09:22 -0700 Subject: [PATCH 07/21] don't announce shard replication at the DHT layer --- lib/persistent.js | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/lib/persistent.js b/lib/persistent.js index ef57d6d6..806f0d0e 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -240,28 +240,21 @@ module.exports = class Persistent { req.reply(phtGetResponse || null) } - async onauthenticatedphtnodeput(req) { + onauthenticatedphtnodeput(req) { if (!req.target || !req.token || !req.value) return - // TODO: authentication! - const unslabbed = unslab(req.value) const p = decode(m.authenticatedPHTNodePutRequest, unslabbed) if (p === null) return const { _publicKey, _topologyID, phtNode, _signature } = p + // TODO: authentication! + // TODO: must we store the whole authenticatedPHTNodePutRequest, or can we discard some of it? const k = b4a.toString(req.target, 'hex') this.authenticatedPHTNodes.set(k, unslabbed) - // TODO: this is the wrong layer of abstraction to do this, because we're announcing shard - // replication but the server is implemented at the hypersearch layer... - if (isLeaf(phtNode)) { - const stream = this.dht.announce(req.target, this.dht.defaultKeyPair) - await stream.finished() - } - req.reply(null) } From e9b0b190dfde4ef91d4ad18bc6ea9c7279658b53 Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Thu, 2 Apr 2026 15:28:33 -0700 Subject: [PATCH 08/21] introduce `connectionKey` for replica discovery --- index.js | 17 +++++++++-------- lib/messages.js | 5 ++++- lib/persistent.js | 12 +++++++----- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/index.js b/index.js index 645948c6..604ba888 100644 --- a/index.js +++ b/index.js @@ -408,25 +408,26 @@ class HyperDHT extends DHT { } async authenticatedPHTNodePut(keyPair, topologyID, phtNode, opts = {}) { + const publicKey = opts.publicKey || keyPair.publicKey + const signAuthenticatedPHTNode = opts.signAuthenticatedPHTNode || Persistent.signAuthenticatedPHTNode const indexID = b4a.allocUnsafe(32) - sodium.crypto_generichash(indexID, b4a.concat([keyPair.publicKey, topologyID])) + sodium.crypto_generichash(indexID, b4a.concat([publicKey, topologyID])) const target = b4a.allocUnsafe(32) // TODO: this re-implements PrefixHashTree._labelHash because we're putting nodes 'a la carte' // outside the context of a PrefixHashTree instance. Is there a better way? sodium.crypto_generichash(target, b4a.from(`${indexID.toString('hex')}${label(phtNode)}`)) - const signature = await signAuthenticatedPHTNode(phtNode, topologyID, keyPair) + const signature = opts.signature || await signAuthenticatedPHTNode(phtNode, topologyID, keyPair) + const connectionKey = opts.connectionKey || b4a.alloc(32) - const signed = c.encode(m.authenticatedPHTNodePutRequest, { - publicKey: keyPair.publicKey, - topologyID: topologyID, - phtNode: phtNode, - signature: signature - }) + const signed = c.encode( + m.authenticatedPHTNodePutRequest, + { publicKey, topologyID, phtNode, signature, connectionKey } + ) opts = { ...opts, diff --git a/lib/messages.js b/lib/messages.js index 0db677a9..ef900c0e 100644 --- a/lib/messages.js +++ b/lib/messages.js @@ -443,19 +443,22 @@ exports.authenticatedPHTNodePutRequest = { c.buffer.preencode(state, m.topologyID) c.json.preencode(state, m.phtNode) c.fixed64.preencode(state, m.signature) + c.fixed32.preencode(state, m.connectionKey) }, encode(state, m) { c.fixed32.encode(state, m.publicKey) c.buffer.encode(state, m.topologyID) c.json.encode(state, m.phtNode) c.fixed64.encode(state, m.signature) + c.fixed32.encode(state, m.connectionKey) }, decode(state) { return { publicKey: c.fixed32.decode(state), topologyID: c.buffer.decode(state), phtNode: c.json.decode(state), - signature: c.fixed64.decode(state) + signature: c.fixed64.decode(state), + connectionKey: c.fixed32.decode(state) } } } diff --git a/lib/persistent.js b/lib/persistent.js index 806f0d0e..d90bd083 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -235,9 +235,12 @@ module.exports = class Persistent { if (!req.target) return const k = b4a.toString(req.target, 'hex') - const phtGetResponse = this.authenticatedPHTNodes.get(k) + const p = this.authenticatedPHTNodes.get(k) - req.reply(phtGetResponse || null) + if (p === null) return req.reply(null) + + const phtGetResponse = c.encode(m.authenticatedPHTNodeGetResponse, p) + req.reply(phtGetResponse) } onauthenticatedphtnodeput(req) { @@ -247,13 +250,12 @@ module.exports = class Persistent { const p = decode(m.authenticatedPHTNodePutRequest, unslabbed) if (p === null) return - const { _publicKey, _topologyID, phtNode, _signature } = p + const { publicKey, topologyID, phtNode, signature, connectionKey } = p // TODO: authentication! - // TODO: must we store the whole authenticatedPHTNodePutRequest, or can we discard some of it? const k = b4a.toString(req.target, 'hex') - this.authenticatedPHTNodes.set(k, unslabbed) + this.authenticatedPHTNodes.set(k, { publicKey, topologyID, phtNode, signature }) req.reply(null) } From f64537b3d48145bf7c682a14a766ad628db58790 Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Fri, 3 Apr 2026 09:28:27 -0700 Subject: [PATCH 09/21] rm control plane document record storage --- lib/constants.js | 4 +--- lib/messages.js | 31 ------------------------------- lib/persistent.js | 27 --------------------------- 3 files changed, 1 insertion(+), 61 deletions(-) diff --git a/lib/constants.js b/lib/constants.js index 06e8ae0a..fb386090 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -12,9 +12,7 @@ const COMMANDS = (exports.COMMANDS = { IMMUTABLE_PUT: 8, IMMUTABLE_GET: 9, AUTHENTICATED_PHT_NODE_PUT: 10, - AUTHENTICATED_PHT_NODE_GET: 11, - PHT_SHARD_PUT: 12, - PHT_SHARD_GET: 13 + AUTHENTICATED_PHT_NODE_GET: 11 }) exports.BOOTSTRAP_NODES = global.Pear?.config.dht?.bootstrap || [ diff --git a/lib/messages.js b/lib/messages.js index ef900c0e..7dc1bb60 100644 --- a/lib/messages.js +++ b/lib/messages.js @@ -484,35 +484,4 @@ exports.authenticatedPHTNodeGetResponse = { signature: c.fixed64.decode(state) } } -} - -exports.phtShardPutRequest = { - preencode(state, m) { - c.buffer.preencode(state, m.key) - c.buffer.preencode(state, m.value) - }, - encode(state, m) { - c.buffer.encode(state, m.key) - c.buffer.encode(state, m.value) - }, - decode(state) { - return { - key: c.buffer.decode(state), - value: c.buffer.decode(state) - } - } -} - -exports.phtShardGetResponse = { - preencode(state, m) { - c.json.preencode(state, m.records) - }, - encode(state, m) { - c.json.encode(state, m.records) - }, - decode(state) { - return { - records: c.json.decode(state) - } - } } \ No newline at end of file diff --git a/lib/persistent.js b/lib/persistent.js index d90bd083..fd6c3f14 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -23,7 +23,6 @@ module.exports = class Persistent { this.mutables = new Cache(opts.mutables) this.immutables = new Cache(opts.immutables) this.authenticatedPHTNodes = new Cache(opts.records) - this.phtShards = new RecordCache(opts.records) } onlookup(req) { @@ -260,32 +259,6 @@ module.exports = class Persistent { req.reply(null) } - onphtshardget(req) { - if (!req.target) return - - // TODO: get and decode the stored PHT node and validate the PHT key against it - - const k = b4a.concat([req.target, req.value]).toString('hex') - - // TODO: parameterize get qty? - const docs = this.phtShards.get(k, 1000) - const encoded = c.encode(m.phtShardGetResponse, { records: docs }) - - req.reply(encoded) - } - - onphtshardput(req) { - if (!req.target || !req.token || !req.value) return - - // TODO: get and decode the stored PHT node and validate the PHT key against it - - const { key, value } = c.decode(m.phtShardPutRequest, unslab(req.value)) - const k = b4a.concat([req.target, key]).toString('hex') - this.phtShards.add(k, value) - - req.reply(null) - } - destroy() { this.records.destroy() this.refreshes.destroy() From c3f378a13f933ef2347dfbaa8695a03b9e4cd4df Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Fri, 3 Apr 2026 09:50:26 -0700 Subject: [PATCH 10/21] rm control plane document record API --- index.js | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/index.js b/index.js index 604ba888..cefd245a 100644 --- a/index.js +++ b/index.js @@ -449,44 +449,6 @@ class HyperDHT extends DHT { return { target: target, indexID: indexID, closestNodes: query.closestNodes } } - async phtShardGet(target, key, opts = {}) { - opts = { ...opts, map: mapPHTShard } - const query = this.query({ target, command: COMMANDS.PHT_SHARD_GET, value: key }, opts) - - for await (const node of query) { - const { value } = node - - // TODO: merge the responses - - return node - } - - return null - } - - async phtShardPut(target, key, value, opts = {}) { - const encoded = c.encode(m.phtShardPutRequest, { - key: key, - value: value - }) - - opts = { - ...opts, - map: mapPHTShard, - commit(reply, dht) { - return dht.request( - { token: reply.token, target, command: COMMANDS.PHT_SHARD_PUT, value: encoded }, - reply.from - ) - } - } - - const query = this.query({ target, command: COMMANDS.PHT_SHARD_GET, value: key }, opts) - await query.finished() - - return { closestNodes: query.closestNodes } - } - onrequest(req) { switch (req.command) { case COMMANDS.PEER_HANDSHAKE: { @@ -542,14 +504,6 @@ class HyperDHT extends DHT { this._persistent.onauthenticatedphtnodeget(req) return true } - case COMMANDS.PHT_SHARD_PUT: { - this._persistent.onphtshardput(req) - return true - } - case COMMANDS.PHT_SHARD_GET: { - this._persistent.onphtshardget(req) - return true - } } return false From 4f8ff2785a9709d43755d300b31dfe7d875dae07 Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Fri, 3 Apr 2026 09:50:56 -0700 Subject: [PATCH 11/21] rm stale keyword-search example --- examples/keyword-search/index.js | 139 --------------------------- examples/keyword-search/package.json | 5 - 2 files changed, 144 deletions(-) delete mode 100644 examples/keyword-search/index.js delete mode 100644 examples/keyword-search/package.json diff --git a/examples/keyword-search/index.js b/examples/keyword-search/index.js deleted file mode 100644 index e63b3ec7..00000000 --- a/examples/keyword-search/index.js +++ /dev/null @@ -1,139 +0,0 @@ -const b4a = require('b4a') -const { PrefixHashTree } = require('prefix-hash-tree') -const { label } = require('prefix-hash-tree/node') -const createTestnet = require('../../testnet.js') - -async function main() { - const testnet = await createTestnet(20, { port: 49739 }) - - // Introducing: Keyword Search (featuring Bob and Alice) - - const bob = testnet.nodes[0] - const alice = testnet.nodes[1] - - // Bob precomputes a static trie topology over a finite vocabulary using a local datastore - const localNodeTable = new AsyncMap() - - const localPHT = new PrefixHashTree({ - bitDomain: 80, - getFunc: localNodeTable.get.bind(localNodeTable), - putFunc: localNodeTable.put.bind(localNodeTable) - }) - - await localPHT.init() - - const coolVocab = ['dog', 'cat', 'bird', 'fish', 'tree', 'human', 'car', 'boat', 'plane', 'train'] - - for (const keyword of coolVocab) { - await localPHT.insert(localPHT.unhashedKeyFrom(keyword), `shard pointer for ${keyword}`) - } - - const topologyID = b4a.from('coolVocab', 'utf8') - - // Bob stores the precomputed trie topology to the DHT, noting the indexID - let indexID - for (const [_, phtNode] of localNodeTable.map) { - const { target, indexID: id, _ } = - await bob.authenticatedPHTNodePut(bob.defaultKeyPair, topologyID, phtNode) - indexID = id - } - - console.log(`Bob created a keyword search index (ID ${indexID.toString('hex')})`) - - // (Bob shares the indexID with Alice...) - - // Now Alice can use that indexID to discover valid keywords - const searchIndex = new PrefixHashTree({ - indexID: indexID.toString('hex'), - bitDomain: 80, - getFunc: async (target) => { - const res = await alice.authenticatedPHTNodeGet(target) - return res === null ? res : res.phtNode - }, - putFunc: null - }) - - const query1 = await searchIndex.prefixQuery(searchIndex.prefixFrom('ca')) - console.log(`Alice discovered keywords ${query1.map(([key, _]) => key).join(', ')}`) - - const query2 = await searchIndex.prefixQuery(searchIndex.prefixFrom('t')) - console.log(`Alice discovered keywords ${query2.map(([key, _]) => key).join(', ')}`) - - // And Alice can add her document records under any valid keyword - const catOnATrainRecord = b4a.from('cat_on_a_train.gif', 'utf8') - const catOnAPlaneRecord = b4a.from('cat_on_a_plane.gif', 'utf8') - - const catKey = searchIndex.unhashedKeyFrom('cat') - const catShard = await searchIndex.searchLeaf(catKey) - const catTarget = searchIndex._labelHash(label(catShard)) - - await alice.phtShardPut(catTarget, catKey, catOnATrainRecord) - console.log(`Alice inserted ${catOnATrainRecord} under ${catKey}`) - - await alice.phtShardPut(catTarget, catKey, catOnAPlaneRecord) - console.log(`Alice inserted ${catOnAPlaneRecord} under ${catKey}`) - - const trainKey = searchIndex.unhashedKeyFrom('train') - const trainShard = await searchIndex.searchLeaf(trainKey) - const trainTarget = searchIndex._labelHash(label(trainShard)) - - await alice.phtShardPut(trainTarget, trainKey, catOnATrainRecord) - console.log(`Alice inserted ${catOnATrainRecord} under ${trainKey}`) - - const planeKey = searchIndex.unhashedKeyFrom('plane') - const planeShard = await searchIndex.searchLeaf(planeKey) - const planeTarget = searchIndex._labelHash(label(planeShard)) - - await alice.phtShardPut(planeTarget, planeKey, catOnAPlaneRecord) - console.log(`Alice inserted ${catOnAPlaneRecord} under ${planeKey}`) - - // Later, Bob performs a search for "cat AND train" - const bobSearchIndex = new PrefixHashTree({ - indexID: indexID.toString('hex'), - bitDomain: 80, - getFunc: async (target) => { - const res = await bob.authenticatedPHTNodeGet(target) - return res === null ? res : res.phtNode - }, - putFunc: null - }) - - const bobCatKey = bobSearchIndex.unhashedKeyFrom('cat') - const bobCatShard = await bobSearchIndex.searchLeaf(bobCatKey) - const bobCatTarget = bobSearchIndex._labelHash(label(bobCatShard)) - - const foundCatDocs = - (await bob.phtShardGet(bobCatTarget, bobCatKey)).value.records.map(record => b4a.from(record).toString('utf8')) - - const bobPlaneKey = bobSearchIndex.unhashedKeyFrom('plane') - const bobPlaneShard = await bobSearchIndex.searchLeaf(bobPlaneKey) - const bobPlaneTarget = bobSearchIndex._labelHash(label(bobPlaneShard)) - - const foundPlaneDocs = - (await bob.phtShardGet(bobPlaneTarget, bobPlaneKey)).value.records.map(record => b4a.from(record).toString('utf8')) - - const intersection = [...new Set(foundCatDocs.filter(x => foundPlaneDocs.includes(x)))] - - console.log(`Bob searched '${bobCatKey} AND ${bobPlaneKey}' and found: ${intersection}`) -} - -class AsyncMap { - constructor() { - this.map = new Map() - } - - async get(key) { - return new Promise((resolve) => { - resolve(this.map.get(key.toString('hex')) || null) - }) - } - - async put(key, val) { - return new Promise((resolve) => { - this.map.set(key.toString('hex'), val) - resolve(val) - }) - } -} - -main() \ No newline at end of file diff --git a/examples/keyword-search/package.json b/examples/keyword-search/package.json deleted file mode 100644 index 1381bbdb..00000000 --- a/examples/keyword-search/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dependencies": { - "b4a": "^1.8.0" - } -} From ac99f527ea5a1c758bfd2ae9a24091598a02698a Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Fri, 3 Apr 2026 10:36:02 -0700 Subject: [PATCH 12/21] refactor `authenticatedPHTNodePut` --- index.js | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index cefd245a..7409201b 100644 --- a/index.js +++ b/index.js @@ -413,13 +413,10 @@ class HyperDHT extends DHT { const signAuthenticatedPHTNode = opts.signAuthenticatedPHTNode || Persistent.signAuthenticatedPHTNode - const indexID = b4a.allocUnsafe(32) - sodium.crypto_generichash(indexID, b4a.concat([publicKey, topologyID])) - const target = b4a.allocUnsafe(32) - - // TODO: this re-implements PrefixHashTree._labelHash because we're putting nodes 'a la carte' - // outside the context of a PrefixHashTree instance. Is there a better way? - sodium.crypto_generichash(target, b4a.from(`${indexID.toString('hex')}${label(phtNode)}`)) + const hash = b4a.allocUnsafe(32) + sodium.crypto_generichash(hash, b4a.concat([publicKey, topologyID])) + const indexID = b4a.toString(hash, 'hex') + const target = new PrefixHashTree({ indexID })._labelHash(label(phtNode)) const signature = opts.signature || await signAuthenticatedPHTNode(phtNode, topologyID, keyPair) const connectionKey = opts.connectionKey || b4a.alloc(32) @@ -446,7 +443,7 @@ class HyperDHT extends DHT { ) await query.finished() - return { target: target, indexID: indexID, closestNodes: query.closestNodes } + return { target, indexID, closestNodes: query.closestNodes } } onrequest(req) { From 16fc1cbc0c05f51e4ad688ed12cbb8f2c5800885 Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Fri, 3 Apr 2026 10:51:15 -0700 Subject: [PATCH 13/21] renaming --- index.js | 8 ++++---- lib/messages.js | 18 +++++++++--------- lib/persistent.js | 8 ++++---- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/index.js b/index.js index 7409201b..28210742 100644 --- a/index.js +++ b/index.js @@ -407,23 +407,23 @@ class HyperDHT extends DHT { return null } - async authenticatedPHTNodePut(keyPair, topologyID, phtNode, opts = {}) { + async authenticatedPHTNodePut(keyPair, treeID, phtNode, opts = {}) { const publicKey = opts.publicKey || keyPair.publicKey const signAuthenticatedPHTNode = opts.signAuthenticatedPHTNode || Persistent.signAuthenticatedPHTNode const hash = b4a.allocUnsafe(32) - sodium.crypto_generichash(hash, b4a.concat([publicKey, topologyID])) + sodium.crypto_generichash(hash, b4a.concat([publicKey, treeID])) const indexID = b4a.toString(hash, 'hex') const target = new PrefixHashTree({ indexID })._labelHash(label(phtNode)) - const signature = opts.signature || await signAuthenticatedPHTNode(phtNode, topologyID, keyPair) + const signature = opts.signature || await signAuthenticatedPHTNode(phtNode, treeID, keyPair) const connectionKey = opts.connectionKey || b4a.alloc(32) const signed = c.encode( m.authenticatedPHTNodePutRequest, - { publicKey, topologyID, phtNode, signature, connectionKey } + { publicKey, treeID, phtNode, signature, connectionKey } ) opts = { diff --git a/lib/messages.js b/lib/messages.js index 7dc1bb60..68155716 100644 --- a/lib/messages.js +++ b/lib/messages.js @@ -422,16 +422,16 @@ exports.mutableGetResponse = { exports.authenticatedPHTNodeSignable = { preencode(state, m) { - c.buffer.preencode(state, m.topologyID) + c.buffer.preencode(state, m.treeID) c.json.preencode(state, m.phtNode) }, encode(state, m) { - c.buffer.encode(state, m.topologyID) + c.buffer.encode(state, m.treeID) c.json.encode(state, m.phtNode) }, decode(state, m) { return { - topologyID: c.buffer.decode(state), + treeID: c.buffer.decode(state), node: c.json.decode(state) } } @@ -440,14 +440,14 @@ exports.authenticatedPHTNodeSignable = { exports.authenticatedPHTNodePutRequest = { preencode(state, m) { c.fixed32.preencode(state, m.publicKey) - c.buffer.preencode(state, m.topologyID) + c.buffer.preencode(state, m.treeID) c.json.preencode(state, m.phtNode) c.fixed64.preencode(state, m.signature) c.fixed32.preencode(state, m.connectionKey) }, encode(state, m) { c.fixed32.encode(state, m.publicKey) - c.buffer.encode(state, m.topologyID) + c.buffer.encode(state, m.treeID) c.json.encode(state, m.phtNode) c.fixed64.encode(state, m.signature) c.fixed32.encode(state, m.connectionKey) @@ -455,7 +455,7 @@ exports.authenticatedPHTNodePutRequest = { decode(state) { return { publicKey: c.fixed32.decode(state), - topologyID: c.buffer.decode(state), + treeID: c.buffer.decode(state), phtNode: c.json.decode(state), signature: c.fixed64.decode(state), connectionKey: c.fixed32.decode(state) @@ -466,20 +466,20 @@ exports.authenticatedPHTNodePutRequest = { exports.authenticatedPHTNodeGetResponse = { preencode(state, m) { c.fixed32.preencode(state, m.publicKey) - c.buffer.preencode(state, m.topologyID) + c.buffer.preencode(state, m.treeID) c.json.preencode(state, m.phtNode) c.fixed64.preencode(state, m.signature) }, encode(state, m) { c.fixed32.encode(state, m.publicKey) - c.buffer.encode(state, m.topologyID) + c.buffer.encode(state, m.treeID) c.json.encode(state, m.phtNode) c.fixed64.encode(state, m.signature) }, decode(state) { return { publicKey: c.fixed32.decode(state), - topologyID: c.buffer.decode(state), + treeID: c.buffer.decode(state), phtNode: c.json.decode(state), signature: c.fixed64.decode(state) } diff --git a/lib/persistent.js b/lib/persistent.js index fd6c3f14..17aa3d25 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -249,12 +249,12 @@ module.exports = class Persistent { const p = decode(m.authenticatedPHTNodePutRequest, unslabbed) if (p === null) return - const { publicKey, topologyID, phtNode, signature, connectionKey } = p + const { publicKey, treeID, phtNode, signature, connectionKey } = p // TODO: authentication! const k = b4a.toString(req.target, 'hex') - this.authenticatedPHTNodes.set(k, { publicKey, topologyID, phtNode, signature }) + this.authenticatedPHTNodes.set(k, { publicKey, treeID, phtNode, signature }) req.reply(null) } @@ -289,13 +289,13 @@ module.exports = class Persistent { return sign(annSignable(target, token, id, ann, NS.UNANNOUNCE), keyPair) } - static signAuthenticatedPHTNode(phtNode, topologyID, keyPair) { + static signAuthenticatedPHTNode(phtNode, treeID, keyPair) { const signable = b4a.allocUnsafe(32 + 32) const hash = signable.subarray(32) signable.set(NS.AUTHENTICATED_PHT_NODE_PUT, 0) - sodium.crypto_generichash(hash, c.encode(m.authenticatedPHTNodeSignable, { topologyID, phtNode })) + sodium.crypto_generichash(hash, c.encode(m.authenticatedPHTNodeSignable, { treeID, phtNode })) return sign(signable, keyPair) } } From fc2646980fd3111eb09d62059e897b3bc2d8b80e Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Fri, 3 Apr 2026 13:45:11 -0700 Subject: [PATCH 14/21] PHT node authentication --- lib/persistent.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/persistent.js b/lib/persistent.js index 17aa3d25..b2081faa 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -4,7 +4,7 @@ const RecordCache = require('record-cache') const Cache = require('xache') const b4a = require('b4a') const unslab = require('unslab') -const { isLeaf } = require('prefix-hash-tree/node') +const { isNode } = require('prefix-hash-tree/node') const { encodeUnslab } = require('./encode') const m = require('./messages') @@ -229,7 +229,6 @@ module.exports = class Persistent { req.reply(null) } - // TODO: does this require any additional sophistication? onauthenticatedphtnodeget(req) { if (!req.target) return @@ -251,7 +250,8 @@ module.exports = class Persistent { const { publicKey, treeID, phtNode, signature, connectionKey } = p - // TODO: authentication! + if (!isNode(phtNode)) return + if (!Persistent.verifyAuthenticatedPHTNode(signature, treeID, phtNode, publicKey)) return const k = b4a.toString(req.target, 'hex') this.authenticatedPHTNodes.set(k, { publicKey, treeID, phtNode, signature }) @@ -298,6 +298,16 @@ module.exports = class Persistent { sodium.crypto_generichash(hash, c.encode(m.authenticatedPHTNodeSignable, { treeID, phtNode })) return sign(signable, keyPair) } + + static verifyAuthenticatedPHTNode(signature, treeID, phtNode, publicKey) { + const signable = b4a.allocUnsafe(32 + 32) + const hash = signable.subarray(32) + + signable.set(NS.AUTHENTICATED_PHT_NODE_PUT, 0) + + sodium.crypto_generichash(hash, c.encode(m.authenticatedPHTNodeSignable, { treeID, phtNode })) + return sodium.crypto_sign_verify_detached(signature, signable, publicKey) + } } function verifyMutable(signature, seq, value, publicKey) { From 1080b2d5b5443003a1151f15ba3dc3fe5481cd47 Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Fri, 3 Apr 2026 14:37:49 -0700 Subject: [PATCH 15/21] PHT node authentication at GET time, renaming and cleanup --- index.js | 66 +++++++++++++++++++++-------------------------- lib/constants.js | 10 +++---- lib/messages.js | 6 ++--- lib/persistent.js | 34 ++++++++++++------------ 4 files changed, 54 insertions(+), 62 deletions(-) diff --git a/index.js b/index.js index 28210742..511a6b7a 100644 --- a/index.js +++ b/index.js @@ -17,7 +17,7 @@ const RawStreamSet = require('./lib/raw-stream-set') const ConnectionPool = require('./lib/connection-pool') const { STREAM_NOT_CONNECTED } = require('./lib/errors') const { PrefixHashTree } = require('prefix-hash-tree') -const { label } = require('prefix-hash-tree/node') +const { label, isNode } = require('prefix-hash-tree/node') const DEFAULTS = { ...DHT.DEFAULTS, @@ -391,54 +391,55 @@ class HyperDHT extends DHT { return { publicKey: keyPair.publicKey, closestNodes: query.closestNodes, seq, signature } } - async authenticatedPHTNodeGet(target, opts = {}) { - opts = { ...opts, map: mapAuthenticatedPHTNode } - const query = this.query({ target, command: COMMANDS.AUTHENTICATED_PHT_NODE_GET, value: null }, opts) + async phtNodeGet(target, opts = {}) { + opts = { ...opts, map: mapPHTNode } - for await (const node of query) { - const { phtNode } = node - - // TODO: implement Persistent.verifyAuthenticatedPHTNode and verify here - // (what else must we verify?) + const query = this.query({ + target, + command: COMMANDS.PHT_NODE_GET, + value: null + }, opts) - return node + for await (const node of query) { + const { publicKey, treeID, phtNode, signature } = node + if (isNode(phtNode) && Persistent.verifyPHTNode(signature, treeID, phtNode, publicKey)) return node } return null } - async authenticatedPHTNodePut(keyPair, treeID, phtNode, opts = {}) { + async phtNodePut(keyPair, treeID, phtNode, opts = {}) { const publicKey = opts.publicKey || keyPair.publicKey - const signAuthenticatedPHTNode = - opts.signAuthenticatedPHTNode || Persistent.signAuthenticatedPHTNode + const signPHTNode = opts.signPHTNode || Persistent.signPHTNode const hash = b4a.allocUnsafe(32) sodium.crypto_generichash(hash, b4a.concat([publicKey, treeID])) const indexID = b4a.toString(hash, 'hex') const target = new PrefixHashTree({ indexID })._labelHash(label(phtNode)) - const signature = opts.signature || await signAuthenticatedPHTNode(phtNode, treeID, keyPair) + const signature = opts.signature || await signPHTNode(phtNode, treeID, keyPair) const connectionKey = opts.connectionKey || b4a.alloc(32) const signed = c.encode( - m.authenticatedPHTNodePutRequest, + m.phtNodePutRequest, { publicKey, treeID, phtNode, signature, connectionKey } ) opts = { ...opts, - map: mapAuthenticatedPHTNode, + map: mapPHTNode + , commit(reply, dht) { return dht.request( - { token: reply.token, target, command: COMMANDS.AUTHENTICATED_PHT_NODE_PUT, value: signed }, + { token: reply.token, target, command: COMMANDS.PHT_NODE_PUT, value: signed }, reply.from ) } } const query = this.query( - { target, command: COMMANDS.AUTHENTICATED_PHT_NODE_GET, value: null }, + { target, command: COMMANDS.PHT_NODE_GET, value: null }, opts ) await query.finished() @@ -493,12 +494,12 @@ class HyperDHT extends DHT { this._persistent.onimmutableget(req) return true } - case COMMANDS.AUTHENTICATED_PHT_NODE_PUT: { - this._persistent.onauthenticatedphtnodeput(req) + case COMMANDS.PHT_NODE_PUT: { + this._persistent.onphtnodeput(req) return true } - case COMMANDS.AUTHENTICATED_PHT_NODE_GET: { - this._persistent.onauthenticatedphtnodeget(req) + case COMMANDS.PHT_NODE_GET: { + this._persistent.onphtnodeget(req) return true } } @@ -645,34 +646,27 @@ function mapMutable(node) { } } -function mapAuthenticatedPHTNode(node) { +function mapPHTNode(node) { if (!node.value) return null try { - const { phtNode } = c.decode(m.authenticatedPHTNodeGetResponse, node.value) + const p = c.decode(m.phtNodeGetResponse, node.value) + const { publicKey, treeID, phtNode, signature } = p return { token: node.token, from: node.from, to: node.to, - phtNode: phtNode + publicKey, + treeID, + phtNode, + signature } } catch { return null } } -function mapPHTShard(node) { - if (!node.value) return null - - return { - token: node.token, - from: node.from, - to: node.to, - value: c.decode(m.phtShardGetResponse, node.value) - } -} - function noop() {} function filterNode(node) { diff --git a/lib/constants.js b/lib/constants.js index fb386090..33d7ba8d 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -11,8 +11,8 @@ const COMMANDS = (exports.COMMANDS = { MUTABLE_GET: 7, IMMUTABLE_PUT: 8, IMMUTABLE_GET: 9, - AUTHENTICATED_PHT_NODE_PUT: 10, - AUTHENTICATED_PHT_NODE_GET: 11 + PHT_NODE_PUT: 10, + PHT_NODE_GET: 11 }) exports.BOOTSTRAP_NODES = global.Pear?.config.dht?.bootstrap || [ @@ -47,7 +47,7 @@ const [ NS_MUTABLE_PUT, NS_PEER_HANDSHAKE, NS_PEER_HOLEPUNCH, - NS_AUTHENTICATED_PHT_NODE_PUT + NS_PHT_NODE_PUT ] = crypto.namespace('hyperswarm/dht', [ COMMANDS.ANNOUNCE, @@ -55,7 +55,7 @@ const [ COMMANDS.MUTABLE_PUT, COMMANDS.PEER_HANDSHAKE, COMMANDS.PEER_HOLEPUNCH, - COMMANDS.AUTHENTICATED_PHT_NODE_PUT + COMMANDS.PHT_NODE_PUT ]) exports.NS = { @@ -64,5 +64,5 @@ exports.NS = { MUTABLE_PUT: NS_MUTABLE_PUT, PEER_HANDSHAKE: NS_PEER_HANDSHAKE, PEER_HOLEPUNCH: NS_PEER_HOLEPUNCH, - AUTHENTICATED_PHT_NODE_PUT: NS_AUTHENTICATED_PHT_NODE_PUT + PHT_NODE_PUT: NS_PHT_NODE_PUT } diff --git a/lib/messages.js b/lib/messages.js index 68155716..77c2b5da 100644 --- a/lib/messages.js +++ b/lib/messages.js @@ -420,7 +420,7 @@ exports.mutableGetResponse = { } } -exports.authenticatedPHTNodeSignable = { +exports.phtNodeSignable = { preencode(state, m) { c.buffer.preencode(state, m.treeID) c.json.preencode(state, m.phtNode) @@ -437,7 +437,7 @@ exports.authenticatedPHTNodeSignable = { } } -exports.authenticatedPHTNodePutRequest = { +exports.phtNodePutRequest = { preencode(state, m) { c.fixed32.preencode(state, m.publicKey) c.buffer.preencode(state, m.treeID) @@ -463,7 +463,7 @@ exports.authenticatedPHTNodePutRequest = { } } -exports.authenticatedPHTNodeGetResponse = { +exports.phtNodeGetResponse = { preencode(state, m) { c.fixed32.preencode(state, m.publicKey) c.buffer.preencode(state, m.treeID) diff --git a/lib/persistent.js b/lib/persistent.js index b2081faa..de1288a2 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -22,7 +22,7 @@ module.exports = class Persistent { this.refreshes = new Cache(opts.refreshes) this.mutables = new Cache(opts.mutables) this.immutables = new Cache(opts.immutables) - this.authenticatedPHTNodes = new Cache(opts.records) + this.phtNodes = new Cache(opts.records) } onlookup(req) { @@ -229,32 +229,30 @@ module.exports = class Persistent { req.reply(null) } - onauthenticatedphtnodeget(req) { + onphtnodeget(req) { if (!req.target) return const k = b4a.toString(req.target, 'hex') - const p = this.authenticatedPHTNodes.get(k) + const p = this.phtNodes.get(k) if (p === null) return req.reply(null) - const phtGetResponse = c.encode(m.authenticatedPHTNodeGetResponse, p) + const phtGetResponse = c.encode(m.phtNodeGetResponse, p) req.reply(phtGetResponse) } - onauthenticatedphtnodeput(req) { + onphtnodeput(req) { if (!req.target || !req.token || !req.value) return const unslabbed = unslab(req.value) - const p = decode(m.authenticatedPHTNodePutRequest, unslabbed) + const p = decode(m.phtNodePutRequest, unslabbed) if (p === null) return const { publicKey, treeID, phtNode, signature, connectionKey } = p - - if (!isNode(phtNode)) return - if (!Persistent.verifyAuthenticatedPHTNode(signature, treeID, phtNode, publicKey)) return - + if (!isNode(phtNode) || !Persistent.verifyPHTNode(signature, treeID, phtNode, publicKey)) return + const k = b4a.toString(req.target, 'hex') - this.authenticatedPHTNodes.set(k, { publicKey, treeID, phtNode, signature }) + this.phtNodes.set(k, { publicKey, treeID, phtNode, signature }) req.reply(null) } @@ -264,7 +262,7 @@ module.exports = class Persistent { this.refreshes.destroy() this.mutables.destroy() this.immutables.destroy() - this.authenticatedPHTNodes.destroy() + this.phtNodes.destroy() } static signMutable(seq, value, keyPair) { @@ -289,23 +287,23 @@ module.exports = class Persistent { return sign(annSignable(target, token, id, ann, NS.UNANNOUNCE), keyPair) } - static signAuthenticatedPHTNode(phtNode, treeID, keyPair) { + static signPHTNode(phtNode, treeID, keyPair) { const signable = b4a.allocUnsafe(32 + 32) const hash = signable.subarray(32) - signable.set(NS.AUTHENTICATED_PHT_NODE_PUT, 0) + signable.set(NS.PHT_NODE_PUT, 0) - sodium.crypto_generichash(hash, c.encode(m.authenticatedPHTNodeSignable, { treeID, phtNode })) + sodium.crypto_generichash(hash, c.encode(m.phtNodeSignable, { treeID, phtNode })) return sign(signable, keyPair) } - static verifyAuthenticatedPHTNode(signature, treeID, phtNode, publicKey) { + static verifyPHTNode(signature, treeID, phtNode, publicKey) { const signable = b4a.allocUnsafe(32 + 32) const hash = signable.subarray(32) - signable.set(NS.AUTHENTICATED_PHT_NODE_PUT, 0) + signable.set(NS.PHT_NODE_PUT, 0) - sodium.crypto_generichash(hash, c.encode(m.authenticatedPHTNodeSignable, { treeID, phtNode })) + sodium.crypto_generichash(hash, c.encode(m.phtNodeSignable, { treeID, phtNode })) return sodium.crypto_sign_verify_detached(signature, signable, publicKey) } } From d1a02a73252c68d15bc81588c6199cb5055d034c Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Fri, 3 Apr 2026 16:30:06 -0700 Subject: [PATCH 16/21] gate behind an experimental flag --- index.js | 10 ++++++++++ lib/persistent.js | 6 +++--- testnet.js | 4 +++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 511a6b7a..2ed0cbbe 100644 --- a/index.js +++ b/index.js @@ -62,6 +62,7 @@ class HyperDHT extends DHT { this._randomPunchInterval = opts.randomPunchInterval || DEFAULTS.randomPunchInterval // min 20s between random punches... this._randomPunches = 0 this._randomPunchLimit = 1 // set to one for extra safety for now + this._experimentalPHT = opts.experimentalPHT === true this.once('persistent', () => { this._persistent = new Persistent(this, persistent) @@ -392,6 +393,8 @@ class HyperDHT extends DHT { } async phtNodeGet(target, opts = {}) { + if (!this._experimentalPHT) return + opts = { ...opts, map: mapPHTNode } const query = this.query({ @@ -409,6 +412,8 @@ class HyperDHT extends DHT { } async phtNodePut(keyPair, treeID, phtNode, opts = {}) { + if (!this._experimentalPHT) return + const publicKey = opts.publicKey || keyPair.publicKey const signPHTNode = opts.signPHTNode || Persistent.signPHTNode @@ -691,6 +696,7 @@ function defaultCacheOpts(opts) { }, relayAddresses: { maxSize: Math.min(maxSize, 512), maxAge: 0 }, persistent: { + experimentalPHT: opts.experimentalPHT, records: { maxSize, maxAge }, refreshes: { maxSize, maxAge }, mutables: { @@ -701,6 +707,10 @@ function defaultCacheOpts(opts) { maxSize: (maxSize / 2) | 0, maxAge: opts.maxAge || 48 * 60 * 60 * 1000 // 48 hours }, + phtNodes: { + maxSize: (maxSize / 2) | 0, + maxAge: opts.maxAge || 48 * 60 * 60 * 1000 // 48 hours + }, bumps: { maxSize, maxAge } } } diff --git a/lib/persistent.js b/lib/persistent.js index de1288a2..7995ced1 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -22,7 +22,7 @@ module.exports = class Persistent { this.refreshes = new Cache(opts.refreshes) this.mutables = new Cache(opts.mutables) this.immutables = new Cache(opts.immutables) - this.phtNodes = new Cache(opts.records) + this.phtNodes = opts.experimentalPHT === true ? new Cache(opts.phtNodes) : null } onlookup(req) { @@ -230,7 +230,7 @@ module.exports = class Persistent { } onphtnodeget(req) { - if (!req.target) return + if (!req.target || !this.phtNodes) return const k = b4a.toString(req.target, 'hex') const p = this.phtNodes.get(k) @@ -242,7 +242,7 @@ module.exports = class Persistent { } onphtnodeput(req) { - if (!req.target || !req.token || !req.value) return + if (!req.target || !req.token || !req.value || !this.phtNodes) return const unslabbed = unslab(req.value) const p = decode(m.phtNodePutRequest, unslabbed) diff --git a/testnet.js b/testnet.js index 7c6cf787..e71aee9a 100644 --- a/testnet.js +++ b/testnet.js @@ -1,6 +1,6 @@ const DHT = require('.') -module.exports = async function createTestnet(size = 10, opts = {}) { +module.exports = async function createTestnet(size = 10, opts = {}, dhtOpts = {}) { const swarm = [] const teardown = typeof opts === 'function' ? opts : opts.teardown ? opts.teardown.bind(opts) : noop @@ -12,6 +12,7 @@ module.exports = async function createTestnet(size = 10, opts = {}) { if (size === 0) return new Testnet(swarm) const first = new DHT({ + ...dhtOpts, ephemeral: false, firewalled: false, bootstrap, @@ -27,6 +28,7 @@ module.exports = async function createTestnet(size = 10, opts = {}) { while (swarm.length < size) { const node = new DHT({ + ...dhtOpts, ephemeral: false, firewalled: false, bootstrap, From 63e5c1fac9a277aacbc8163b65927ea1150dcfda Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Mon, 6 Apr 2026 09:01:14 -0700 Subject: [PATCH 17/21] prettier --- index.js | 43 +++++++++++++++++++++++-------------------- lib/constants.js | 17 ++++++++--------- lib/messages.js | 2 +- lib/persistent.js | 6 +++--- 4 files changed, 35 insertions(+), 33 deletions(-) diff --git a/index.js b/index.js index 2ed0cbbe..f4a82841 100644 --- a/index.js +++ b/index.js @@ -17,7 +17,7 @@ const RawStreamSet = require('./lib/raw-stream-set') const ConnectionPool = require('./lib/connection-pool') const { STREAM_NOT_CONNECTED } = require('./lib/errors') const { PrefixHashTree } = require('prefix-hash-tree') -const { label, isNode } = require('prefix-hash-tree/node') +const { label, isNode } = require('prefix-hash-tree/node') const DEFAULTS = { ...DHT.DEFAULTS, @@ -397,15 +397,19 @@ class HyperDHT extends DHT { opts = { ...opts, map: mapPHTNode } - const query = this.query({ - target, - command: COMMANDS.PHT_NODE_GET, - value: null - }, opts) + const query = this.query( + { + target, + command: COMMANDS.PHT_NODE_GET, + value: null + }, + opts + ) for await (const node of query) { const { publicKey, treeID, phtNode, signature } = node - if (isNode(phtNode) && Persistent.verifyPHTNode(signature, treeID, phtNode, publicKey)) return node + if (isNode(phtNode) && Persistent.verifyPHTNode(signature, treeID, phtNode, publicKey)) + return node } return null @@ -415,7 +419,7 @@ class HyperDHT extends DHT { if (!this._experimentalPHT) return const publicKey = opts.publicKey || keyPair.publicKey - + const signPHTNode = opts.signPHTNode || Persistent.signPHTNode const hash = b4a.allocUnsafe(32) @@ -423,18 +427,20 @@ class HyperDHT extends DHT { const indexID = b4a.toString(hash, 'hex') const target = new PrefixHashTree({ indexID })._labelHash(label(phtNode)) - const signature = opts.signature || await signPHTNode(phtNode, treeID, keyPair) + const signature = opts.signature || (await signPHTNode(phtNode, treeID, keyPair)) const connectionKey = opts.connectionKey || b4a.alloc(32) - const signed = c.encode( - m.phtNodePutRequest, - { publicKey, treeID, phtNode, signature, connectionKey } - ) + const signed = c.encode(m.phtNodePutRequest, { + publicKey, + treeID, + phtNode, + signature, + connectionKey + }) opts = { ...opts, - map: mapPHTNode - , + map: mapPHTNode, commit(reply, dht) { return dht.request( { token: reply.token, target, command: COMMANDS.PHT_NODE_PUT, value: signed }, @@ -443,10 +449,7 @@ class HyperDHT extends DHT { } } - const query = this.query( - { target, command: COMMANDS.PHT_NODE_GET, value: null }, - opts - ) + const query = this.query({ target, command: COMMANDS.PHT_NODE_GET, value: null }, opts) await query.finished() return { target, indexID, closestNodes: query.closestNodes } @@ -656,7 +659,7 @@ function mapPHTNode(node) { try { const p = c.decode(m.phtNodeGetResponse, node.value) - const { publicKey, treeID, phtNode, signature } = p + const { publicKey, treeID, phtNode, signature } = p return { token: node.token, diff --git a/lib/constants.js b/lib/constants.js index 33d7ba8d..12d282ab 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -48,15 +48,14 @@ const [ NS_PEER_HANDSHAKE, NS_PEER_HOLEPUNCH, NS_PHT_NODE_PUT -] = - crypto.namespace('hyperswarm/dht', [ - COMMANDS.ANNOUNCE, - COMMANDS.UNANNOUNCE, - COMMANDS.MUTABLE_PUT, - COMMANDS.PEER_HANDSHAKE, - COMMANDS.PEER_HOLEPUNCH, - COMMANDS.PHT_NODE_PUT - ]) +] = crypto.namespace('hyperswarm/dht', [ + COMMANDS.ANNOUNCE, + COMMANDS.UNANNOUNCE, + COMMANDS.MUTABLE_PUT, + COMMANDS.PEER_HANDSHAKE, + COMMANDS.PEER_HOLEPUNCH, + COMMANDS.PHT_NODE_PUT +]) exports.NS = { ANNOUNCE: NS_ANNOUNCE, diff --git a/lib/messages.js b/lib/messages.js index 77c2b5da..cd26f866 100644 --- a/lib/messages.js +++ b/lib/messages.js @@ -484,4 +484,4 @@ exports.phtNodeGetResponse = { signature: c.fixed64.decode(state) } } -} \ No newline at end of file +} diff --git a/lib/persistent.js b/lib/persistent.js index 7995ced1..152d2064 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -236,7 +236,7 @@ module.exports = class Persistent { const p = this.phtNodes.get(k) if (p === null) return req.reply(null) - + const phtGetResponse = c.encode(m.phtNodeGetResponse, p) req.reply(phtGetResponse) } @@ -248,9 +248,9 @@ module.exports = class Persistent { const p = decode(m.phtNodePutRequest, unslabbed) if (p === null) return - const { publicKey, treeID, phtNode, signature, connectionKey } = p + const { publicKey, treeID, phtNode, signature, connectionKey } = p if (!isNode(phtNode) || !Persistent.verifyPHTNode(signature, treeID, phtNode, publicKey)) return - + const k = b4a.toString(req.target, 'hex') this.phtNodes.set(k, { publicKey, treeID, phtNode, signature }) From b5f4f9df138cc2e685cff2f8b4530dcfb4af03a9 Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Mon, 6 Apr 2026 09:03:56 -0700 Subject: [PATCH 18/21] gate persistent `phtNodes` destructor behind experimental flag --- lib/persistent.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/persistent.js b/lib/persistent.js index 152d2064..c6ac9539 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -262,7 +262,7 @@ module.exports = class Persistent { this.refreshes.destroy() this.mutables.destroy() this.immutables.destroy() - this.phtNodes.destroy() + if (this.phtNodes) this.phtNodes.destroy() } static signMutable(seq, value, keyPair) { From e346234a2680fe6d1e040738d4d7d93b8ed6b49b Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Mon, 6 Apr 2026 12:19:46 -0700 Subject: [PATCH 19/21] add tests --- index.js | 2 +- test/helpers/index.js | 4 ++-- test/storing.js | 28 ++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index f4a82841..e84a5562 100644 --- a/index.js +++ b/index.js @@ -452,7 +452,7 @@ class HyperDHT extends DHT { const query = this.query({ target, command: COMMANDS.PHT_NODE_GET, value: null }, opts) await query.finished() - return { target, indexID, closestNodes: query.closestNodes } + return { target, indexID, closestNodes: query.closestNodes, signature, connectionKey } } onrequest(req) { diff --git a/test/helpers/index.js b/test/helpers/index.js index 658a53c8..a89a4f85 100644 --- a/test/helpers/index.js +++ b/test/helpers/index.js @@ -12,8 +12,8 @@ async function toArray(iterable) { return result } -async function swarm(t, n = 32, bootstrap = []) { - return createTestnet(n, { bootstrap, teardown: t.teardown }) +async function swarm(t, n = 32, bootstrap = [], dhtOpts = {}) { + return createTestnet(n, { bootstrap, teardown: t.teardown }, dhtOpts) } async function* spawnFixture(t, args) { diff --git a/test/storing.js b/test/storing.js index 13949382..85462500 100644 --- a/test/storing.js +++ b/test/storing.js @@ -1,6 +1,7 @@ const test = require('brittle') const HyperDHT = require('../') const { swarm } = require('./helpers') +const { createPHTNode } = require('prefix-hash-tree/node') test('immutable put - get', async function (t) { const { nodes } = await swarm(t, 100) @@ -62,3 +63,30 @@ test('mutable put - put - get', async function (t) { t.is(Buffer.compare(res.signature, put2.signature), 0) t.is(res.value.toString(), 'testing two') }) + +test('PHT node put - get', async function (t) { + const { nodes } = await swarm(t, 100, [], { experimentalPHT: true }) + const keyPair = HyperDHT.keyPair() + const n = createPHTNode({ label: '010011', child0: '0100110', child1: '0100111' }) + + const p = await nodes[30].phtNodePut(keyPair, Buffer.from('myTestTree'), n) + + t.is(p.target.length, 32) + t.is(p.signature.length, 64) + t.is(typeof p.indexID, 'string') + t.is(Buffer.compare(Buffer.alloc(32), p.connectionKey), 0) + + const res = await nodes[30].phtNodeGet(p.target) + const { publicKey, treeID, phtNode, signature } = res + + t.is(Buffer.compare(publicKey, keyPair.publicKey), 0) + t.is(treeID.toString(), 'myTestTree') + t.alike(phtNode, n) + t.is(Buffer.compare(signature, p.signature), 0) + t.is(typeof res.from, 'object') + t.is(typeof res.from.host, 'string') + t.is(typeof res.from.port, 'number') + t.is(typeof res.to, 'object') + t.is(typeof res.to.host, 'string') + t.is(typeof res.to.port, 'number') +}) From 50a7719631cdd28514a3b5b245215f6ffa3119d8 Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Mon, 6 Apr 2026 13:57:36 -0700 Subject: [PATCH 20/21] verify target --- index.js | 12 +++++++++++- lib/persistent.js | 14 ++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index e84a5562..ac10e37a 100644 --- a/index.js +++ b/index.js @@ -408,7 +408,17 @@ class HyperDHT extends DHT { for await (const node of query) { const { publicKey, treeID, phtNode, signature } = node - if (isNode(phtNode) && Persistent.verifyPHTNode(signature, treeID, phtNode, publicKey)) + + const hash = b4a.allocUnsafe(32) + sodium.crypto_generichash(hash, b4a.concat([publicKey, treeID])) + const indexID = b4a.toString(hash, 'hex') + const t = new PrefixHashTree({ indexID })._labelHash(label(phtNode)) + + if ( + isNode(phtNode) && + t.equals(target) && + Persistent.verifyPHTNode(signature, treeID, phtNode, publicKey) + ) return node } diff --git a/lib/persistent.js b/lib/persistent.js index c6ac9539..424b5084 100644 --- a/lib/persistent.js +++ b/lib/persistent.js @@ -4,7 +4,8 @@ const RecordCache = require('record-cache') const Cache = require('xache') const b4a = require('b4a') const unslab = require('unslab') -const { isNode } = require('prefix-hash-tree/node') +const { PrefixHashTree } = require('prefix-hash-tree') +const { label, isNode } = require('prefix-hash-tree/node') const { encodeUnslab } = require('./encode') const m = require('./messages') @@ -249,7 +250,16 @@ module.exports = class Persistent { if (p === null) return const { publicKey, treeID, phtNode, signature, connectionKey } = p - if (!isNode(phtNode) || !Persistent.verifyPHTNode(signature, treeID, phtNode, publicKey)) return + + if (!isNode(phtNode)) return + + const hash = b4a.allocUnsafe(32) + sodium.crypto_generichash(hash, b4a.concat([publicKey, treeID])) + const indexID = b4a.toString(hash, 'hex') + const t = new PrefixHashTree({ indexID })._labelHash(label(phtNode)) + + if (!t.equals(req.target) || !Persistent.verifyPHTNode(signature, treeID, phtNode, publicKey)) + return const k = b4a.toString(req.target, 'hex') this.phtNodes.set(k, { publicKey, treeID, phtNode, signature }) From faae40e91299630269ebde48805a52ab39d8f0ae Mon Sep 17 00:00:00 2001 From: Noah Levenson Date: Mon, 6 Apr 2026 19:33:08 -0700 Subject: [PATCH 21/21] deps --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 165bab25..42487e4a 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "hypercore-id-encoding": "^1.2.0", "noise-curve-ed": "^2.0.0", "noise-handshake": "^4.0.0", + "prefix-hash-tree": "^0.0.2", "record-cache": "^1.1.1", "safety-catch": "^1.0.1", "signal-promise": "^1.0.3",