From d2d9f37270f265d6db580daf8cab38843bda38a9 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 2 Aug 2023 11:02:20 +0200 Subject: [PATCH] wip --- fetcher/content-claims/api.js | 0 fetcher/content-claims/api.ts | 19 ++ fetcher/content-claims/block-batch.js | 47 +++++ fetcher/content-claims/block-fetcher.js | 182 ++++++++++++++++++++ fetcher/content-claims/car.js | 19 ++ fetcher/content-claims/dag-index.js | 163 ++++++++++++++++++ fetcher/content-claims/http-data-fetcher.js | 9 + fetcher/content-claims/index.js | 0 package-lock.json | 175 ++++++++++++++++++- package.json | 3 + 10 files changed, 611 insertions(+), 6 deletions(-) create mode 100644 fetcher/content-claims/api.js create mode 100644 fetcher/content-claims/api.ts create mode 100644 fetcher/content-claims/block-batch.js create mode 100644 fetcher/content-claims/block-fetcher.js create mode 100644 fetcher/content-claims/car.js create mode 100644 fetcher/content-claims/dag-index.js create mode 100644 fetcher/content-claims/http-data-fetcher.js create mode 100644 fetcher/content-claims/index.js diff --git a/fetcher/content-claims/api.js b/fetcher/content-claims/api.js new file mode 100644 index 0000000..e69de29 diff --git a/fetcher/content-claims/api.ts b/fetcher/content-claims/api.ts new file mode 100644 index 0000000..9054deb --- /dev/null +++ b/fetcher/content-claims/api.ts @@ -0,0 +1,19 @@ +import { UnknownLink } from 'multiformats/link' +import { MultihashIndexItem } from 'cardex/multihash-index-sorted/api' +import { CARLink } from 'cardex/api' + +export interface IndexEntry extends MultihashIndexItem { + origin: CARLink +} + +export interface Index { + get (c: UnknownLink): Promise +} + +export interface DataFetcherStreamOptions { + range?: { offset: number, length?: number } +} + +export interface DataFetcher { + stream (k: string, options?: DataFetcherStreamOptions): Promise> +} diff --git a/fetcher/content-claims/block-batch.js b/fetcher/content-claims/block-batch.js new file mode 100644 index 0000000..147d0ba --- /dev/null +++ b/fetcher/content-claims/block-batch.js @@ -0,0 +1,47 @@ +const MAX_BYTES_BETWEEN = 1024 * 1024 * 2 +const MAX_BATCH_SIZE = 10 + +/** + * @typedef {import('multiformats').UnknownLink} UnknownLink + * @typedef {{ carCid: import('cardex/api').CARLink, blockCid: UnknownLink, offset: number }} BatchItem + * @typedef {{ add: (i: BatchItem) => void, remove: (cid: UnknownLink) => void, next: () => BatchItem[] }} BlockBatcher + */ + +/** + * Batcher for blocks in CARs. Batches are grouped by CAR CID and blocks are + * returned in batches in the order they were inserted. + * @implements {BlockBatcher} + */ +export class OrderedCarBlockBatcher { + /** @type {BatchItem[]} */ + #queue = [] + + /** @param {BatchItem} item */ + add (item) { + this.#queue.push(item) + } + + /** @param {UnknownLink} cid */ + remove (cid) { + this.#queue = this.#queue.filter(item => item.blockCid.toString() !== cid.toString()) + } + + next () { + const queue = this.#queue + let prevItem = queue.shift() + if (!prevItem) return [] + const batch = [prevItem] + while (true) { + const item = queue.at(0) + if (!item) break + if (item.carCid.toString() !== prevItem.carCid.toString() || item.offset - prevItem.offset >= MAX_BYTES_BETWEEN) { + break + } + batch.push(item) + queue.shift() // remove from the queue + if (batch.length >= MAX_BATCH_SIZE) break + prevItem = item + } + return batch + } +} diff --git a/fetcher/content-claims/block-fetcher.js b/fetcher/content-claims/block-fetcher.js new file mode 100644 index 0000000..871a87a --- /dev/null +++ b/fetcher/content-claims/block-fetcher.js @@ -0,0 +1,182 @@ +import { readBlockHead, asyncIterableReader } from '@ipld/car/decoder' +import { base58btc } from 'multiformats/bases/base58' +import defer from 'p-defer' +import { OrderedCarBlockBatcher } from './block-batch.js' + +/** + * @typedef {import('multiformats').UnknownLink} UnknownLink + * @typedef {import('dagula').Block} Block + * @typedef {import('@cloudflare/workers-types').R2Bucket} R2Bucket + */ + +// 2MB (max safe libp2p block size) + typical block header length + some leeway +const MAX_ENCODED_BLOCK_LENGTH = (1024 * 1024 * 2) + 39 + 61 + +export class BlockFetcher { + /** + * @param {R2Bucket} dataBucket + * @param {import('./dag-index/api.js').Index} index + */ + constructor (dataBucket, index) { + this._dataBucket = dataBucket + this._idx = index + } + + /** @param {UnknownLink} cid */ + async get (cid) { + // console.log(`get ${cid}`) + const entry = await this._idx.get(cid) + if (!entry) return + const carPath = `${entry.origin}/${entry.origin}.car` + const range = { offset: entry.offset } + const res = await this._dataBucket.get(carPath, { range }) + if (!res) return + + const reader = res.body.getReader() + const bytesReader = asyncIterableReader((async function * () { + while (true) { + const { done, value } = await reader.read() + if (done) return + yield value + } + })()) + + const blockHeader = await readBlockHead(bytesReader) + const bytes = await bytesReader.exactly(blockHeader.blockLength) + reader.cancel() + return { cid, bytes } + } +} + +export class BatchingBlockFetcher extends BlockFetcher { + /** @type {Map>>} */ + #pendingBlocks = new Map() + + /** @type {import('./block-batch').BlockBatcher} */ + #batcher = new OrderedCarBlockBatcher() + + #scheduled = false + + /** @type {Promise|null} */ + #processing = null + + #scheduleBatchProcessing () { + if (this.#scheduled) return + this.#scheduled = true + + const startProcessing = async () => { + this.#scheduled = false + const { promise, resolve } = defer() + this.#processing = promise + try { + await this.#processBatch() + } finally { + this.#processing = null + resolve() + } + } + + // If already running, then start when finished + if (this.#processing) { + return this.#processing.then(startProcessing) + } + + // If not running, then start on the next tick + setTimeout(startProcessing) + } + + async #processBatch () { + console.log('processing batch') + const batcher = this.#batcher + this.#batcher = new OrderedCarBlockBatcher() + const pendingBlocks = this.#pendingBlocks + this.#pendingBlocks = new Map() + + while (true) { + const batch = batcher.next() + if (!batch.length) break + + batch.sort((a, b) => a.offset - b.offset) + + const { carCid } = batch[0] + const carPath = `${carCid}/${carCid}.car` + const range = { + offset: batch[0].offset, + length: batch[batch.length - 1].offset - batch[0].offset + MAX_ENCODED_BLOCK_LENGTH + } + + console.log(`fetching ${batch.length} blocks from ${carCid} (${range.length} bytes @ ${range.offset})`) + const res = await this._dataBucket.get(carPath, { range }) + if (!res) { + // should not happen, but if it does, we need to resolve `undefined` + // for the blocks in this batch - they are not found. + for (const blocks of pendingBlocks.values()) { + blocks.forEach(b => b.resolve()) + } + return + } + + const reader = res.body.getReader() + const bytesReader = asyncIterableReader((async function * () { + while (true) { + const { done, value } = await reader.read() + if (done) return + yield value + } + })()) + + while (true) { + try { + const blockHeader = await readBlockHead(bytesReader) + const bytes = await bytesReader.exactly(blockHeader.blockLength) + bytesReader.seek(blockHeader.blockLength) + + const key = mhToKey(blockHeader.cid.multihash.bytes) + const blocks = pendingBlocks.get(key) + if (blocks) { + // console.log(`got wanted block for ${blockHeader.cid}`) + const block = { cid: blockHeader.cid, bytes } + blocks.forEach(b => b.resolve(block)) + pendingBlocks.delete(key) + // remove from batcher if queued to be read + batcher.remove(blockHeader.cid) + } + } catch { + break + } + } + // we should have read all the bytes from the reader by now but if the + // bytesReader throws for bad data _before_ the end then we need to + // cancel the reader - we don't need the rest. + reader.cancel() + } + + // resolve `undefined` for any remaining blocks + for (const blocks of pendingBlocks.values()) { + blocks.forEach(b => b.resolve()) + } + } + + /** @param {UnknownLink} cid */ + async get (cid) { + // console.log(`get ${cid}`) + const entry = await this._idx.get(cid) + if (!entry) return + + this.#batcher.add({ carCid: entry.origin, blockCid: cid, offset: entry.offset }) + + if (!entry.multihash) throw new Error('missing entry multihash') + const key = mhToKey(entry.multihash.bytes) + let blocks = this.#pendingBlocks.get(key) + if (!blocks) { + blocks = [] + this.#pendingBlocks.set(key, blocks) + } + const deferred = defer() + blocks.push(deferred) + this.#scheduleBatchProcessing() + return deferred.promise + } +} + +const mhToKey = (/** @type {Uint8Array} */ mh) => base58btc.encode(mh) diff --git a/fetcher/content-claims/car.js b/fetcher/content-claims/car.js new file mode 100644 index 0000000..551e075 --- /dev/null +++ b/fetcher/content-claims/car.js @@ -0,0 +1,19 @@ +import { CarReader } from '@ipld/car' + +export const code = 0x0202 + +/** + * @param {Uint8Array} bytes + * @returns {Promise>} + */ +export async function decode (bytes) { + const reader = await CarReader.fromBytes(bytes) + const blocks = [] + for await (const b of reader.blocks()) { + blocks.push({ + cid: /** @type {import('multiformats').UnknownLink} */ (b.cid), + bytes: b.bytes + }) + } + return blocks +} diff --git a/fetcher/content-claims/dag-index.js b/fetcher/content-claims/dag-index.js new file mode 100644 index 0000000..a2eebe0 --- /dev/null +++ b/fetcher/content-claims/dag-index.js @@ -0,0 +1,163 @@ +/* global ReadableStream */ +import debug from 'debug' +import * as Link from 'multiformats/link' +import * as raw from 'multiformats/codecs/raw' +import * as Claims from '@web3-storage/content-claims/client' +import { MultihashIndexSortedReader } from 'cardex/multihash-index-sorted' +import { Map as LinkMap } from 'lnmap' +import * as CAR from './car.js' + +/** + * @typedef {import('multiformats').UnknownLink} UnknownLink + * @typedef {import('./api').IndexEntry} IndexEntry + * @typedef {import('./api').Index} Index + */ + +const log = debug('dagula:fetcher:content-claims:dag-index') + +/** @implements {Index} */ +export class ContentClaimsIndex { + /** + * Index store. + * @type {import('../../bindings').SimpleBucket} + */ + #bucket + /** + * Cached index entries. + * @type {Map} + */ + #cache + /** + * CIDs for which we have already fetched claims. + * + * Note: _only_ the CIDs which have been explicitly queried, for which we + * have made a content claim request. Not using `this.#cache` because reading + * a claim may cause us to add other CIDs to the cache that we haven't read + * claims for. + * + * Note: implemented as a Map not a Set so that we take advantage of the + * key cache that `lnmap` provides, so we don't duplicate base58 encoded + * multihash keys. + * @type {Map} + */ + #claimFetched + /** + * @type {URL|undefined} + */ + #serviceURL + + /** + * @param {import('../../bindings').SimpleBucket} bucket Bucket that stores CARs. + * @param {{ serviceURL?: URL }} [options] + */ + constructor (bucket, options) { + this.#bucket = bucket + this.#cache = new LinkMap() + this.#claimFetched = new LinkMap() + this.#serviceURL = options?.serviceURL + } + + /** + * @param {UnknownLink} cid + * @returns {Promise} + */ + async get (cid) { + // get the index data for this CID (CAR CID & offset) + let indexItem = this.#cache.get(cid) + + // read the index for _this_ CID to get the index data for it's _links_. + // + // when we get to the bottom of the tree (raw blocks), we want to be able + // to send back the index information without having to read claims for + // each leaf. We can only do that if we read the claims for the parent now. + if (indexItem) { + // we found the index data! ...if this CID is raw, then there's no links + // and no more index information to discover so don't read claims. + if (cid.code !== raw.code) { + await this.#readClaims(cid) + } + } else { + // we not found the index data! + await this.#readClaims(cid) + // seeing as we just read the index for this CID we _should_ have some + // index information for it now. + indexItem = this.#cache.get(cid) + // if not then, well, it's not found! + if (!indexItem) return + } + return indexItem + } + + /** + * Read claims for the passed CID and populate the cache. + * @param {import('multiformats').UnknownLink} cid + */ + async #readClaims (cid) { + if (this.#claimFetched.has(cid)) return + + const claims = await Claims.read(cid, { serviceURL: this.#serviceURL }) + for (const claim of claims) { + // skip anything that is not a relation claim, since we know by + // our naming convention that our CAR files are named after their hash + // and we don't serve anything that we don't have in our own bucket. + if (claim.type !== 'assert/relation') continue + + // export the blocks from the claim - may include the CARv2 indexes + const blocks = [...claim.export()] + + // each part is a tuple of CAR CID (content) & CARv2 index CID (includes) + for (const { content, includes } of claim.parts) { + if (!isCARLink(content)) continue + if (!includes) continue + + /** @type {{ cid: import('multiformats').UnknownLink, bytes: Uint8Array }|undefined} */ + let block = blocks.find(b => b.cid.toString() === includes.content.toString()) + + // if the index is not included in the claim, it should be in CARPARK + if (!block && includes.parts?.length) { + const obj = await this.#bucket.get(`${includes.parts[0]}/${includes.parts[0]}.car`) + if (!obj) continue + const blocks = await CAR.decode(new Uint8Array(await obj.arrayBuffer())) + block = blocks.find(b => b.cid.toString() === includes.content.toString()) + } + if (!block) continue + + const entries = await decodeIndex(content, block.bytes) + for (const entry of entries) { + this.#cache.set(Link.create(raw.code, entry.multihash), entry) + } + } + break + } + this.#claimFetched.set(cid, true) + } +} + +/** + * @param {import('multiformats').Link} cid + * @returns {cid is import('cardex/api').CARLink} + */ +const isCARLink = cid => cid.code === CAR.code + +/** + * Read a MultihashIndexSorted index for the passed origin CAR and return a + * list of IndexEntry. + * @param {import('cardex/api').CARLink} origin + * @param {Uint8Array} bytes + */ +const decodeIndex = async (origin, bytes) => { + const entries = [] + const readable = new ReadableStream({ + pull (controller) { + controller.enqueue(bytes) + controller.close() + } + }) + const reader = MultihashIndexSortedReader.createReader({ reader: readable.getReader() }) + while (true) { + const { done, value } = await reader.read() + if (done) break + entries.push(/** @type {IndexEntry} */({ origin, ...value })) + } + return entries +} diff --git a/fetcher/content-claims/http-data-fetcher.js b/fetcher/content-claims/http-data-fetcher.js new file mode 100644 index 0000000..8f56da8 --- /dev/null +++ b/fetcher/content-claims/http-data-fetcher.js @@ -0,0 +1,9 @@ +/** + * @typedef {import('./api').DataFetcher} DataFetcher + * @implements {DataFetcher} + */ +class HTTPDataFetcher { + stream (url, options) { + + } +} diff --git a/fetcher/content-claims/index.js b/fetcher/content-claims/index.js new file mode 100644 index 0000000..e69de29 diff --git a/package-lock.json b/package-lock.json index 9b446e9..bba858f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "dagula", - "version": "6.0.2", + "version": "7.0.0", "license": "Apache-2.0 OR MIT", "dependencies": { "@chainsafe/libp2p-noise": "^11.0.0", @@ -24,13 +24,16 @@ "@libp2p/websockets": "^5.0.3", "@multiformats/blake2": "^1.0.13", "@multiformats/multiaddr": "^11.3.0", + "@web3-storage/content-claims": "^3.0.1", "archy": "^1.0.0", + "cardex": "^2.3.1", "conf": "^11.0.1", "debug": "^4.3.4", "ipfs-unixfs-exporter": "^12.0.2", "it-length-prefixed": "^8.0.4", "it-pipe": "^2.0.3", "libp2p": "^0.42.2", + "lnmap": "^1.0.1", "multiformats": "^11.0.1", "p-defer": "^4.0.0", "protobufjs": "^7.0.0", @@ -405,18 +408,35 @@ } }, "node_modules/@ipld/dag-cbor": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.0.0.tgz", - "integrity": "sha512-zdsiSiYDEOIDW7mmWOYWC9gukjXO+F8wqxz/LfN7iSwTfIyipC8+UQrCbPupFMRb/33XQTZk8yl3My8vUQBRoA==", + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.0.3.tgz", + "integrity": "sha512-A2UFccS0+sARK9xwXiVZIaWbLbPxLGP3UZOjBeOMWfDY04SXi8h1+t4rHBzOlKYF/yWNm3RbFLyclWO7hZcy4g==", "dependencies": { - "cborg": "^1.10.0", - "multiformats": "^11.0.0" + "cborg": "^2.0.1", + "multiformats": "^12.0.1" }, "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" } }, + "node_modules/@ipld/dag-cbor/node_modules/cborg": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-2.0.3.tgz", + "integrity": "sha512-f1IbyqgRLQK4ruNM+V3WikfYfXQg/f/zC1oneOw1P7F/Dn2OJX6MaXIdei3JMpz361IjY7OENBKcE53nkJFVCQ==", + "bin": { + "cborg": "cli.js" + } + }, + "node_modules/@ipld/dag-cbor/node_modules/multiformats": { + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.0.1.tgz", + "integrity": "sha512-s01wijBJoDUqESWSzePY0lvTw7J3PVO9x2Cc6ASI5AMZM2Gnhh7BC17+nlFhHKU7dDzaCaRfb+NiqNzOsgPUoQ==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/@ipld/dag-json": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/@ipld/dag-json/-/dag-json-10.0.1.tgz", @@ -442,6 +462,16 @@ "npm": ">=7.0.0" } }, + "node_modules/@ipld/dag-ucan": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/@ipld/dag-ucan/-/dag-ucan-3.3.2.tgz", + "integrity": "sha512-EhuOrAfnudsVYIbzEIgi3itHAEo3WZNOt1VNPsYhxKBhOzDMeoTXh6/IHc7ZKBW1T2vDQHdgj4m1r64z6MssGA==", + "dependencies": { + "@ipld/dag-cbor": "^9.0.0", + "@ipld/dag-json": "^10.0.0", + "multiformats": "^11.0.0" + } + }, "node_modules/@ipld/unixfs": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/@ipld/unixfs/-/unixfs-2.1.1.tgz", @@ -2064,6 +2094,67 @@ "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.1.tgz", "integrity": "sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==" }, + "node_modules/@ucanto/client": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@ucanto/client/-/client-8.0.0.tgz", + "integrity": "sha512-2nuAzdcFawPzqQZO/7MvdtOkkxdS5OLLDmKbPV8aEV7Xo7QzwvO3gfrR1sC967ggf+O758LhaMPEXczcSspHbQ==", + "dependencies": { + "@ucanto/core": "^8.0.0", + "@ucanto/interface": "^8" + } + }, + "node_modules/@ucanto/core": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@ucanto/core/-/core-8.0.0.tgz", + "integrity": "sha512-Ne45bH0uUhAexNkJWj8tYsvKut58LaL3rsu30vxVh+ybkj47nD02mY8KqW+vCHY80E1ubPcjlHuCt0+efR9bgw==", + "dependencies": { + "@ipld/car": "^5.1.0", + "@ipld/dag-cbor": "^9.0.0", + "@ipld/dag-ucan": "^3.3.2", + "@ucanto/interface": "^8.0.0", + "multiformats": "^11.0.0" + } + }, + "node_modules/@ucanto/interface": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@ucanto/interface/-/interface-8.0.0.tgz", + "integrity": "sha512-xeJJYdGAPKOYbCiG8BsGmyoBovZDtVya+42Gtd8fViZeNSS3h0f2BPDBS91YFOxSGswqCd2fqvrfrlg3TTMmZw==", + "dependencies": { + "@ipld/dag-ucan": "^3.3.2", + "multiformats": "^11.0.0" + } + }, + "node_modules/@ucanto/server": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/@ucanto/server/-/server-8.0.1.tgz", + "integrity": "sha512-QgtEUgJwXmmbFA4cdQ0kOhaSPekAdVNl9vYrE59Joy6OhEh5fvnOiGMYWdkExFmvL4v1DhKZwVH8p9iQ+eWFTw==", + "dependencies": { + "@ucanto/core": "^8", + "@ucanto/interface": "^8", + "@ucanto/validator": "^8" + } + }, + "node_modules/@ucanto/transport": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@ucanto/transport/-/transport-8.0.0.tgz", + "integrity": "sha512-1/3ELUtUAu7zNKq/jy/PjvvCoFt7cFS8198E8U04UM1EsZS0WvK2VnEa/aS8MA5bBy8TjRFmQtXjcndZ0RE5vw==", + "dependencies": { + "@ucanto/core": "^8.0.0", + "@ucanto/interface": "^8" + } + }, + "node_modules/@ucanto/validator": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@ucanto/validator/-/validator-8.0.0.tgz", + "integrity": "sha512-S+cGKUVu074TT1FaoOyZa3mKf3CuEBLHLlE3TU1UoIC5Yp9WnvX+cDOGKIyfJ/HgHHBvAEDxYNOkNZbCATsRdA==", + "dependencies": { + "@ipld/car": "^5.1.0", + "@ipld/dag-cbor": "^9.0.0", + "@ucanto/core": "^8.0.0", + "@ucanto/interface": "^8", + "multiformats": "^11.0.0" + } + }, "node_modules/@web-std/stream": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@web-std/stream/-/stream-1.0.1.tgz", @@ -2073,6 +2164,27 @@ "web-streams-polyfill": "^3.1.1" } }, + "node_modules/@web3-storage/content-claims": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@web3-storage/content-claims/-/content-claims-3.0.1.tgz", + "integrity": "sha512-7+JLmQaw4mDgs+VREtkHtk3Q0Bp1rv4Kt5YN2Qw/g73xvRja28/Uioy7iv83YHez52lAPwXFkSkGb0ifJy5PLw==", + "dependencies": { + "@ucanto/client": "^8.0.0", + "@ucanto/server": "^8.0.1", + "@ucanto/transport": "^8.0.0", + "carstream": "^1.0.2", + "multiformats": "^12.0.1" + } + }, + "node_modules/@web3-storage/content-claims/node_modules/multiformats": { + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.0.1.tgz", + "integrity": "sha512-s01wijBJoDUqESWSzePY0lvTw7J3PVO9x2Cc6ASI5AMZM2Gnhh7BC17+nlFhHKU7dDzaCaRfb+NiqNzOsgPUoQ==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/abortable-iterator": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/abortable-iterator/-/abortable-iterator-4.0.3.tgz", @@ -2557,6 +2669,40 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/cardex": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/cardex/-/cardex-2.3.1.tgz", + "integrity": "sha512-850vVrRGg48z3aPuEmPvDjQBs+Bp4shSEF8Smmr412NvhCJ7ka2kBQzruoBF6FrotexGxftvmILRH3gUTZQmpQ==", + "dependencies": { + "@ipld/car": "^5.1.0", + "multiformats": "^11.0.2", + "sade": "^1.8.1", + "uint8arrays": "^4.0.3", + "varint": "^6.0.0" + }, + "bin": { + "cardex": "src/bin.js" + } + }, + "node_modules/carstream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/carstream/-/carstream-1.1.0.tgz", + "integrity": "sha512-tbf8FOnGX1+0kOe77nm9MG53REiqQopDwzwbXYVxUcsKOAHG2KSD++qy95v1vrtRt1Q6L0Sb01it7QwJ+Yt1sQ==", + "dependencies": { + "@ipld/dag-cbor": "^9.0.3", + "multiformats": "^12.0.1", + "uint8arraylist": "^2.4.3" + } + }, + "node_modules/carstream/node_modules/multiformats": { + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.0.1.tgz", + "integrity": "sha512-s01wijBJoDUqESWSzePY0lvTw7J3PVO9x2Cc6ASI5AMZM2Gnhh7BC17+nlFhHKU7dDzaCaRfb+NiqNzOsgPUoQ==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/cbor": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/cbor/-/cbor-8.1.0.tgz", @@ -5693,6 +5839,23 @@ "uint8arraylist": "^2.3.2" } }, + "node_modules/lnmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lnmap/-/lnmap-1.0.1.tgz", + "integrity": "sha512-AmgSSYqvPHWCqMpJAZGRdkY82FXBRHwSVVf25uGCqge7xO4oabef7E4XzNUzverVhL5YobnjkgzgvidfXbfMBA==", + "dependencies": { + "multiformats": "^12.0.1" + } + }, + "node_modules/lnmap/node_modules/multiformats": { + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.0.1.tgz", + "integrity": "sha512-s01wijBJoDUqESWSzePY0lvTw7J3PVO9x2Cc6ASI5AMZM2Gnhh7BC17+nlFhHKU7dDzaCaRfb+NiqNzOsgPUoQ==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/load-json-file": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-7.0.1.tgz", diff --git a/package.json b/package.json index 4c935f4..1036bfc 100644 --- a/package.json +++ b/package.json @@ -32,13 +32,16 @@ "@libp2p/websockets": "^5.0.3", "@multiformats/blake2": "^1.0.13", "@multiformats/multiaddr": "^11.3.0", + "@web3-storage/content-claims": "^3.0.1", "archy": "^1.0.0", + "cardex": "^2.3.1", "conf": "^11.0.1", "debug": "^4.3.4", "ipfs-unixfs-exporter": "^12.0.2", "it-length-prefixed": "^8.0.4", "it-pipe": "^2.0.3", "libp2p": "^0.42.2", + "lnmap": "^1.0.1", "multiformats": "^11.0.1", "p-defer": "^4.0.0", "protobufjs": "^7.0.0",