feat(api): add HTTP Range request support for streaming

.github/actions/ci-setup/action.yml

@@ -16,11 +16,10 @@ runs:
   using: "composite"
   steps:
       - name: Install Nim
-        uses: iffy/install-nim@v5
+        uses: jiro4989/setup-nim-action@v2
         with:
-          version: binary:${{ inputs.nim_version }}
-        env:
-          GITHUB_TOKEN: ${{ inputs.github_token }}
+          nim-version: ${{ inputs.nim_version }}
+          repo-token: ${{ inputs.github_token }}
 
       - name: Setup CMake
         uses: lukka/get-cmake@latest

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  nim_version:   2.2.4
+  nim_version:   2.2.6
   rust_version:  1.79.0
   cmake_version: 3.x
   node_version:  22

.github/workflows/release.yml

@@ -15,7 +15,7 @@ on:
   workflow_dispatch:
 
 env:
-  nim_version:   2.2.4
+  nim_version:   2.2.6
   rust_version:  1.79.0
   cmake_version: 3.x
   archivist_binary_base: archivist

.gitignore

@@ -41,6 +41,10 @@ docker/hostdatadir
 docker/prometheus-data
 .DS_Store
 
+.claude
+.mcp.json
+
 data/
 nimbledeps
 logs/
+

README.md

@@ -14,7 +14,7 @@
 ### Prerequisites
 
 The following should be installed before building the node:
-- [Nim][nim] 2.2.4
+- [Nim][nim] 2.2.6
 - [Cmake][cmake] 3.x
 - [Rust][rustup] 1.79.0
 - Optional: [NodeJS][nodejs] 22.x, only required for some tests

archivist.nimble

@@ -18,8 +18,8 @@ requires "https://github.com/durability-labs/nim-serde >= 2.1.0"
 requires "https://github.com/durability-labs/nim-leopard >= 0.2.2"
 requires "https://github.com/guzba/zippy >= 0.10.16"
 requires "https://github.com/durability-labs/nim-chronicles#version-0-12-3-pre" # TODO: update to version 0.12.3 once it is released
-requires "https://github.com/durability-labs/nim-groth16 >= 0.1.0"
-requires "https://github.com/durability-labs/circom-witnessgen >= 0.1.3"
+requires "https://github.com/durability-labs/nim-groth16 >= 0.1.1"
+requires "https://github.com/durability-labs/circom-witnessgen >= 0.1.4"
 
 import std/os
 

archivist/archivisttypes.nim

@@ -42,6 +42,7 @@ const
   Pos2Bn128MrklCodec* = multiCodec("poseidon2-alt_bn_128-merkle-2kb")
 
   ManifestCodec* = multiCodec("codex-manifest")
+  DirectoryCodec* = MultiCodec(0xCD04) # codex-directory (not yet registered in libp2p)
   DatasetRootCodec* = multiCodec("codex-root")
   BlockCodec* = multiCodec("codex-block")
   SlotRootCodec* = multiCodec("codex-slot-root")
@@ -51,8 +52,8 @@ const
   HashesCodecs* = [Sha256HashCodec, Pos2Bn128SpngCodec, Pos2Bn128MrklCodec]
 
   PrimitivesCodecs* = [
-    ManifestCodec, DatasetRootCodec, BlockCodec, SlotRootCodec, SlotProvingRootCodec,
-    SlotCellCodec,
+    ManifestCodec, DirectoryCodec, DatasetRootCodec, BlockCodec, SlotRootCodec,
+    SlotProvingRootCodec, SlotCellCodec,
   ]
 
 proc initEmptyCidTable(): ?!Table[(CidVersion, MultiCodec, MultiCodec), Cid] =

archivist/conf.nim

@@ -334,9 +334,9 @@ type
         "the whole slot id space and the value of " &
         "the --validator-group-index parameter will be ignored. " &
         "Powers of twos are advised for even distribution",
-      defaultValue: ValidationGroups.none,
+      defaultValue: int.none,
       name: "validator-groups"
-    .}: Option[ValidationGroups]
+    .}: Option[int]
 
     validatorGroupIndex* {.
       desc: "Slot validation group index",
@@ -608,6 +608,16 @@ proc readValue*(
 ) {.raises: [SerializationError, IOError].} =
   val = EthAddress.init(r.readValue(string)).get()
 
+proc readValue*(
+    r: var TomlReader, val: var Cid
+) {.raises: [SerializationError, IOError].} =
+  let cidStr = r.readValue(string)
+  let cidResult = Cid.init(cidStr)
+  if cidResult.isOk:
+    val = cidResult.get()
+  else:
+    raise newException(SerializationError, "Invalid CID: " & cidStr)
+
 proc readValue*(r: var TomlReader, val: var SignedPeerRecord) =
   without uri =? r.readValue(string).catch, err:
     error "invalid SignedPeerRecord configuration value", error = err.msg

archivist/directorynode.nim

@@ -0,0 +1,168 @@
+## Copyright (c) 2025 Archivist Authors
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+## Directory operations for ArchivistNode
+##
+## This module is kept separate from node.nim to avoid importing minprotobuf
+## into the main compilation path, which triggers serialization conflicts
+## with TOML config loading.
+
+{.push raises: [].}
+
+import pkg/chronos
+import pkg/questionable
+import pkg/questionable/results
+import pkg/libp2p/cid
+
+# Import minprotobuf explicitly to avoid leaking symbols that conflict with serialization
+from pkg/libp2p/protobuf/minprotobuf import
+  ProtoBuffer, initProtoBuffer, getField, getRequiredRepeatedField,
+  write, finish, ProtoResult
+
+import ./manifest/directory
+import ./blocktype as bt
+import ./stores
+import ./archivisttypes
+import ./units
+import ./errors
+import ./logutils
+
+logScope:
+  topics = "archivist directorynode"
+
+proc decodeDirectoryManifest*(blk: bt.Block): ?!DirectoryManifest =
+  ## Decode a directory manifest from a block
+  ##
+  if not ?blk.cid.isDirectory:
+    return failure "Cid not a directory codec"
+
+  var
+    pbNode = initProtoBuffer(blk.data)
+    pbEntries: seq[seq[byte]]
+    totalSize: uint64
+    name: string
+    entries: seq[DirectoryEntry]
+
+  if pbNode.getRequiredRepeatedField(1, pbEntries).isErr:
+    return failure("Unable to decode `entries` from directory manifest!")
+
+  if pbNode.getField(2, totalSize).isErr:
+    return failure("Unable to decode `totalSize` from directory manifest!")
+
+  if pbNode.getField(3, name).isErr:
+    return failure("Unable to decode `name` from directory manifest!")
+
+  for pbEntryData in pbEntries:
+    var
+      pbEntry = initProtoBuffer(pbEntryData)
+      entryName: string
+      cidBuf: seq[byte]
+      size: uint64
+      isDir: uint32
+      mimetype: string
+
+    if pbEntry.getField(1, entryName).isErr:
+      return failure("Unable to decode entry `name` from directory manifest!")
+
+    if pbEntry.getField(2, cidBuf).isErr:
+      return failure("Unable to decode entry `cid` from directory manifest!")
+
+    if pbEntry.getField(3, size).isErr:
+      return failure("Unable to decode entry `size` from directory manifest!")
+
+    if pbEntry.getField(4, isDir).isErr:
+      return failure("Unable to decode entry `isDirectory` from directory manifest!")
+
+    if pbEntry.getField(5, mimetype).isErr:
+      return failure("Unable to decode entry `mimetype` from directory manifest!")
+
+    let entryCid = ?Cid.init(cidBuf).mapFailure
+
+    entries.add(DirectoryEntry(
+      name: entryName,
+      cid: entryCid,
+      size: size.NBytes,
+      isDirectory: isDir != 0,
+      mimetype: mimetype,
+    ))
+
+  success DirectoryManifest(
+    entries: entries,
+    totalSize: totalSize.NBytes,
+    name: name,
+  )
+
+proc encodeDirectoryManifest*(directory: DirectoryManifest): seq[byte] =
+  ## Encode a directory manifest to protobuf bytes
+  ##
+  var pbNode = initProtoBuffer()
+
+  for entry in directory.entries:
+    var pbEntry = initProtoBuffer()
+    pbEntry.write(1, entry.name)
+    pbEntry.write(2, entry.cid.data.buffer)
+    pbEntry.write(3, entry.size.uint64)
+    pbEntry.write(4, entry.isDirectory.uint32)
+    if entry.mimetype.len > 0:
+      pbEntry.write(5, entry.mimetype)
+    pbEntry.finish()
+    pbNode.write(1, pbEntry)
+
+  pbNode.write(2, directory.totalSize.uint64)
+
+  if directory.name.len > 0:
+    pbNode.write(3, directory.name)
+
+  pbNode.finish()
+  pbNode.buffer
+
+proc storeDirectoryManifest*(
+    networkStore: NetworkStore, directory: DirectoryManifest
+): Future[?!bt.Block] {.async: (raises: [CancelledError]).} =
+  ## Store a directory manifest and return its block
+  ##
+  let encoded = encodeDirectoryManifest(directory)
+
+  without blk =? bt.Block.new(data = encoded, codec = DirectoryCodec), error:
+    trace "Unable to create block from directory manifest"
+    return failure(error)
+
+  if err =? (await networkStore.putBlock(blk)).errorOption:
+    trace "Unable to store directory manifest block", cid = blk.cid, err = err.msg
+    return failure(err)
+
+  info "Stored directory manifest",
+    cid = blk.cid,
+    entries = directory.entries.len,
+    totalSize = directory.totalSize
+
+  success blk
+
+proc fetchDirectoryManifest*(
+    networkStore: NetworkStore, cid: Cid
+): Future[?!DirectoryManifest] {.async: (raises: [CancelledError]).} =
+  ## Fetch and decode a directory manifest block
+  ##
+  if err =? cid.isDirectory.errorOption:
+    return failure "CID has invalid content type for directory manifest {$cid}"
+
+  trace "Retrieving directory manifest for cid", cid
+
+  without blk =? await networkStore.getBlock(BlockAddress.init(cid)), err:
+    trace "Error retrieving directory manifest block", cid, err = err.msg
+    return failure err
+
+  trace "Decoding directory manifest for cid", cid
+
+  without directory =? decodeDirectoryManifest(blk), err:
+    trace "Unable to decode as directory manifest", err = err.msg
+    return failure("Unable to decode as directory manifest")
+
+  trace "Decoded directory manifest", cid, entries = directory.entries.len
+
+  return directory.success

archivist/discovery.nim

@@ -18,6 +18,7 @@ import pkg/libp2p/[cid, multicodec, routing_record, signed_envelope]
 import pkg/questionable
 import pkg/questionable/results
 import pkg/contractabi/address as ca
+import pkg/datastore
 import pkg/archivistdht/discv5/[routing_table, protocol as discv5]
 from pkg/nimcrypto import keccak256
 
@@ -219,7 +220,7 @@ proc new*(
     bindPort = 0.Port,
     announceAddrs: openArray[MultiAddress],
     bootstrapNodes: openArray[SignedPeerRecord] = [],
-    store: Datastore = SQLiteDatastore.new(Memory).expect("Should not fail!"),
+    store: Datastore = SQLiteDatastore.new(datastore.Memory).expect("Should not fail!"),
 ): Discovery =
   ## Create a new Discovery node instance for the given key and datastore
   ##