From 4a7a648f5e791a84b7b07d96d070e23769e0f8de Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Wed, 15 Mar 2023 15:16:01 -0300 Subject: [PATCH 01/14] initial structure --- README.md | 71 ------------------------- book/src/SUMMARY.md | 16 ++++++ book/src/advanced/README.md | 1 + book/src/advanced/swarm_mode.md | 4 ++ book/src/guides/README.md | 1 + book/src/guides/nodejs.md | 3 ++ book/src/guides/python.md | 15 ++++++ book/src/installation/README.md | 7 +++ book/src/installation/binary_release.md | 1 + book/src/installation/docker.md | 1 + book/src/installation/from_source.md | 10 ++++ book/src/reducers/README.md | 1 + book/src/sources/README.md | 1 + book/src/sources/n2c.md | 1 + book/src/sources/n2n.md | 1 + book/src/storage/README.md | 1 + book/src/usage/README.md | 1 + book/src/usage/example.md | 42 +++++++++++++++ 18 files changed, 107 insertions(+), 71 deletions(-) create mode 100644 book/src/advanced/README.md create mode 100644 book/src/advanced/swarm_mode.md create mode 100644 book/src/guides/README.md create mode 100644 book/src/guides/nodejs.md create mode 100644 book/src/guides/python.md create mode 100644 book/src/installation/README.md create mode 100644 book/src/installation/binary_release.md create mode 100644 book/src/installation/docker.md create mode 100644 book/src/installation/from_source.md create mode 100644 book/src/reducers/README.md create mode 100644 book/src/sources/README.md create mode 100644 book/src/sources/n2c.md create mode 100644 book/src/sources/n2n.md create mode 100644 book/src/storage/README.md create mode 100644 book/src/usage/README.md create mode 100644 book/src/usage/example.md diff --git a/README.md b/README.md index affaa8ef..ccd2cef7 100644 --- a/README.md +++ b/README.md @@ -141,59 +141,6 @@ We currently provide the following ways to install _Scrolls_: - By compiling from source code using the instructions provided in this README. -## Configuration - -This is an example configuration file: - -```toml -# get data from a relay node -[source] -type = "N2N" -address = "relays-new.cardano-mainnet.iohk.io:3001" - -# You can optionally enable enrichment (local db with transactions), this is needed for some reducers -[enrich] -type = "Sled" -db_path = "/opt/scrolls/sled_db" - -# enable the "UTXO by Address" collection -[[reducers]] -type = "UtxoByAddress" -# you can optionally prefix the keys in the collection -key_prefix = "c1" -# you can optionally only process UTXO from a set of predetermined addresses -filter = ["addr1qy8jecz3nal788f8t2zy6vj2l9ply3trpnkn2xuvv5rgu4m7y853av2nt8wc33agu3kuakvg0kaee0tfqhgelh2eeyyqgxmxw3"] - -# enable the "Point by Tx" collection -[[reducers]] -type = "PointByTx" -key_prefix = "c2" - -# store the collections in a local Redis -[storage] -type = "Redis" -connection_params = "redis://127.0.0.1:6379" - -# start reading from an arbitrary point in the chain -[intersect] -type = "Point" -value = [57867490, "c491c5006192de2c55a95fb3544f60b96bd1665accaf2dfa2ab12fc7191f016b"] - -# let Scrolls know that we're working with mainnet -[chain] -type = "Mainnet" -``` - -## Compiling from Source - -To compile from source, you'll need to have the Rust toolchain available in your development box. Execute the following command to clone and build the project: - -```sh -git clone https://github.com/txpipe/scrolls.git -cd scrolls -cargo build -``` - ## FAQ ### Don't we have tools for this already? @@ -211,24 +158,6 @@ Yes, we do. We have excellent tools such as: [Kupo](https://github.com/CardanoSo There's some overlap between _Oura_ and _Scrolls_. Both tools read on-chain data and output some data results. The main difference is that Oura is meant to **_react_** to events, to watch the chain and actuate upon certain patterns. In contrast, _Scrolls_ is meant to provide a snapshot of the current state of the chain by aggregating the whole history. They were built to work well together. For example, let's say that you're building an app that uses Oura to process transaction data, you could then integrate _Scrolls_ as a way to lookup the source address of the transaction's input. - -### How do I read the data using Python? - -Assuming you're using Redis as a storage backend (only one available ATM), we recommend using [redis-py](https://github.com/redis/redis-py) package to talk directly to the Redis instance. This is a very simple code snippet to query a the UTXOs by address. - -```python ->>> import redis ->>> r = redis.Redis(host='localhost', port=6379, db=0) ->>> r.smembers("c1.addr1w8tqqyccvj7402zns2tea78d42etw520fzvf22zmyasjdtsv3e5rz") -{b'2548228522837ea580bc55a3e6a09479deca499b5e7f3c08602a1f3191a178e7:20', b'04086c503512833c7a0c11fc85f7d0f0422db9d14b31275b3d4327c40c6fd73b:25'} -``` - - The Redis operation being used is `smembers` which return the list of members of a set stored under a particular key. In this case, we query by the value `c1.addr1w8tqqyccvj7402zns2tea78d42etw520fzvf22zmyasjdtsv3e5rz`, where `c1` is the key prefix specified in the config for our particular collection and `addr1w8tqqyccvj7402zns2tea78d42etw520fzvf22zmyasjdtsv3e5rz` is the address we're interested in querying. The response from redis is the list of UTXOs (in the format `{tx-hash}:{output-index}`) that are associated with that particular address. - -### How do I read the data using NodeJS? - -TODO - ### What is "swarm mode"? Swarm mode is a way to speed up the process of rebuilding collection from scratch by splitting the tasks into concurrent instances of the _Scrolls_ daemon by partitioning the history of the chain into smaller fragments. diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index e0ff9f90..ed473db0 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -1,3 +1,19 @@ # Summary - [Introduction](./introduction.md) +- [Installation](./installation/README.md) + - [Binary Release](./installation/binary_release.md) + - [From Source](./installation/from_source.md) + - [Docker](./installation/docker.md) +- [Usage](./usage/README.md) + - [Example Configuration](./installation/binary_release.md) +- [Sources](./sources/README.md) + - [Node-to-Node](./sources/n2n.md) + - [Node-to-Client](./sources/n2c.md) +- [Reducers](./reducers/README.md) +- [Storage](./storage/README.md) +- [Advanced Features](./advanced/README.md) + - [Swarm Mode](./advanced/swarm_mode.md) +- [Guides](./guides/README.md) + - [NodeJS](./guides/nodejs.md) + - [Python](./guides/python.md) diff --git a/book/src/advanced/README.md b/book/src/advanced/README.md new file mode 100644 index 00000000..7c935bda --- /dev/null +++ b/book/src/advanced/README.md @@ -0,0 +1 @@ +# Advanced features diff --git a/book/src/advanced/swarm_mode.md b/book/src/advanced/swarm_mode.md new file mode 100644 index 00000000..4ea185a4 --- /dev/null +++ b/book/src/advanced/swarm_mode.md @@ -0,0 +1,4 @@ +# Swarm mode + +Swarm mode is a way to speed up the process of rebuilding collection from scratch by splitting the tasks into concurrent instances of the Scrolls daemon by partitioning the history of the chain into smaller fragments. + diff --git a/book/src/guides/README.md b/book/src/guides/README.md new file mode 100644 index 00000000..50a86734 --- /dev/null +++ b/book/src/guides/README.md @@ -0,0 +1 @@ +# Guides diff --git a/book/src/guides/nodejs.md b/book/src/guides/nodejs.md new file mode 100644 index 00000000..f48fa8b1 --- /dev/null +++ b/book/src/guides/nodejs.md @@ -0,0 +1,3 @@ +# NodeJS + +## Read the data using NodeJS diff --git a/book/src/guides/python.md b/book/src/guides/python.md new file mode 100644 index 00000000..c961114a --- /dev/null +++ b/book/src/guides/python.md @@ -0,0 +1,15 @@ +# Python + +## Read the data using Python + +Assuming you're using Redis as a storage backend (only one available ATM), we recommend using [redis-py](https://github.com/redis/redis-py) package to talk directly to the Redis instance. This is a very simple code snippet to query a the UTXOs by address. + +```python +>>> import redis +>>> r = redis.Redis(host='localhost', port=6379, db=0) +>>> r.smembers("c1.addr1w8tqqyccvj7402zns2tea78d42etw520fzvf22zmyasjdtsv3e5rz") +{b'2548228522837ea580bc55a3e6a09479deca499b5e7f3c08602a1f3191a178e7:20', b'04086c503512833c7a0c11fc85f7d0f0422db9d14b31275b3d4327c40c6fd73b:25'} +``` + + The Redis operation being used is `smembers` which return the list of members of a set stored under a particular key. In this case, we query by the value `c1.addr1w8tqqyccvj7402zns2tea78d42etw520fzvf22zmyasjdtsv3e5rz`, where `c1` is the key prefix specified in the config for our particular collection and `addr1w8tqqyccvj7402zns2tea78d42etw520fzvf22zmyasjdtsv3e5rz` is the address we're interested in querying. The response from redis is the list of UTXOs (in the format `{tx-hash}:{output-index}`) that are associated with that particular address. + diff --git a/book/src/installation/README.md b/book/src/installation/README.md new file mode 100644 index 00000000..936c662c --- /dev/null +++ b/book/src/installation/README.md @@ -0,0 +1,7 @@ +# Installation + +Depending on your needs, Scrolls provides different installation options: + +- [Binary Release](binary_release.md): to use one of our pre-compiled binary releases for the supported platforms. +- [From Source](from_source.md): to compile a binary from source code using Rust's toolchain +- [Docker](docker.md): to run the tool from a pre-built docker image diff --git a/book/src/installation/binary_release.md b/book/src/installation/binary_release.md new file mode 100644 index 00000000..16f58931 --- /dev/null +++ b/book/src/installation/binary_release.md @@ -0,0 +1 @@ +# Binary release diff --git a/book/src/installation/docker.md b/book/src/installation/docker.md new file mode 100644 index 00000000..c597eaa5 --- /dev/null +++ b/book/src/installation/docker.md @@ -0,0 +1 @@ +# Docker diff --git a/book/src/installation/from_source.md b/book/src/installation/from_source.md new file mode 100644 index 00000000..d696eb04 --- /dev/null +++ b/book/src/installation/from_source.md @@ -0,0 +1,10 @@ +# From Source + +To compile from source, you'll need to have the Rust toolchain available in your development box. Execute the following command to clone and build the project: + +```sh +git clone https://github.com/txpipe/scrolls.git +cd scrolls +cargo build +``` + diff --git a/book/src/reducers/README.md b/book/src/reducers/README.md new file mode 100644 index 00000000..840b86d4 --- /dev/null +++ b/book/src/reducers/README.md @@ -0,0 +1 @@ +# Reducers diff --git a/book/src/sources/README.md b/book/src/sources/README.md new file mode 100644 index 00000000..e70306d4 --- /dev/null +++ b/book/src/sources/README.md @@ -0,0 +1 @@ +# Sources diff --git a/book/src/sources/n2c.md b/book/src/sources/n2c.md new file mode 100644 index 00000000..f3e76af2 --- /dev/null +++ b/book/src/sources/n2c.md @@ -0,0 +1 @@ +# Node-to-Client diff --git a/book/src/sources/n2n.md b/book/src/sources/n2n.md new file mode 100644 index 00000000..51a2b802 --- /dev/null +++ b/book/src/sources/n2n.md @@ -0,0 +1 @@ +# Node-to-Node diff --git a/book/src/storage/README.md b/book/src/storage/README.md new file mode 100644 index 00000000..e0e8ac41 --- /dev/null +++ b/book/src/storage/README.md @@ -0,0 +1 @@ +# Storage diff --git a/book/src/usage/README.md b/book/src/usage/README.md new file mode 100644 index 00000000..8f04b05a --- /dev/null +++ b/book/src/usage/README.md @@ -0,0 +1 @@ +# Usage diff --git a/book/src/usage/example.md b/book/src/usage/example.md new file mode 100644 index 00000000..74e131df --- /dev/null +++ b/book/src/usage/example.md @@ -0,0 +1,42 @@ +## Example Configuration + +This is an example configuration file: + +```toml +# get data from a relay node +[source] +type = "N2N" +address = "relays-new.cardano-mainnet.iohk.io:3001" + +# You can optionally enable enrichment (local db with transactions), this is needed for some reducers +[enrich] +type = "Sled" +db_path = "/opt/scrolls/sled_db" + +# enable the "UTXO by Address" collection +[[reducers]] +type = "UtxoByAddress" +# you can optionally prefix the keys in the collection +key_prefix = "c1" +# you can optionally only process UTXO from a set of predetermined addresses +filter = ["addr1qy8jecz3nal788f8t2zy6vj2l9ply3trpnkn2xuvv5rgu4m7y853av2nt8wc33agu3kuakvg0kaee0tfqhgelh2eeyyqgxmxw3"] + +# enable the "Point by Tx" collection +[[reducers]] +type = "PointByTx" +key_prefix = "c2" + +# store the collections in a local Redis +[storage] +type = "Redis" +connection_params = "redis://127.0.0.1:6379" + +# start reading from an arbitrary point in the chain +[intersect] +type = "Point" +value = [57867490, "c491c5006192de2c55a95fb3544f60b96bd1665accaf2dfa2ab12fc7191f016b"] + +# let Scrolls know that we're working with mainnet +[chain] +type = "Mainnet" +``` From 97f76413bc979bf6c8dd51f12558d97275051509 Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Wed, 15 Mar 2023 15:26:20 -0300 Subject: [PATCH 02/14] Move installation options out of main README.md --- README.md | 13 +++---------- book/src/installation/binary_release.md | 3 +++ book/src/installation/docker.md | 2 ++ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ccd2cef7..49b980fd 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@
-## Intro +## Introduction _Scrolls_ is a tool for building and maintaining read-optimized collections of Cardano's on-chain entities. It crawls the history of the chain and aggregates all data to reflect the current state of affairs. Once the whole history has been processed, _Scrolls_ watches the tip of the chain to keep the collections up-to-date. @@ -16,6 +16,8 @@ Examples of collections are: "utxo by address", "chain parameters by epoch", "po > In other words, _Scrolls_ is just a map-reduce algorithm that aggregates the history of the chain into use-case-specific, key-value dictionaries. +Check our [documentation](https://txpipe.github.io/scrolls) for detailed information on how to start working with Scrolls. + :warning: this tool is under heavy development. Library API, configuration schema and storage structure may vary drastically. Several important features are still missing. Use at your own peril. ## Storage @@ -132,15 +134,6 @@ Once you're done with the testdive, you can clean your environment by running: docker-compose down ``` -## Installing - -We currently provide the following ways to install _Scrolls_: - -- Using one of the pre-compiled binaries shared via [Github Releases](https://github.com/txpipe/scrolls/releases) -- Using the Docker image shared via [Github Packages](https://github.com/txpipe/scrolls/pkgs/container/scrolls) -- By compiling from source code using the instructions provided in this README. - - ## FAQ ### Don't we have tools for this already? diff --git a/book/src/installation/binary_release.md b/book/src/installation/binary_release.md index 16f58931..035451a0 100644 --- a/book/src/installation/binary_release.md +++ b/book/src/installation/binary_release.md @@ -1 +1,4 @@ # Binary release + + +Installation using one of the pre-compiled binaries shared via [Github Releases](https://github.com/txpipe/scrolls/releases) diff --git a/book/src/installation/docker.md b/book/src/installation/docker.md index c597eaa5..f87417eb 100644 --- a/book/src/installation/docker.md +++ b/book/src/installation/docker.md @@ -1 +1,3 @@ # Docker + +Installation using the Docker image shared via [Github Packages](https://github.com/txpipe/scrolls/pkgs/container/scrolls) From d20782728213da2ab15cc8b3a19e5a1eb6a02e6b Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Wed, 15 Mar 2023 15:37:14 -0300 Subject: [PATCH 03/14] Move testrive out of main file --- README.md | 31 ------------------------------- book/src/SUMMARY.md | 8 +++++--- book/src/guides/nodejs.md | 2 +- book/src/guides/python.md | 2 +- book/src/guides/testdrive.md | 31 +++++++++++++++++++++++++++++++ 5 files changed, 38 insertions(+), 36 deletions(-) create mode 100644 book/src/guides/testdrive.md diff --git a/README.md b/README.md index 49b980fd..15756b42 100644 --- a/README.md +++ b/README.md @@ -103,37 +103,6 @@ Scrolls is a pipeline that takes block data as input and outputs DB update comma - [ ] By Mint Policy / Asset - [ ] By Pool -## Testdrive - -In the `testdrive` folder you'll find a minimal example that uses docker-compose to spin up a local Redis instance and a Scrolls daemon. You'll need Docker and docker-compose installed in your local machine. Run the following commands to start it: - -```sh -cd testdrive -docker-compose up -``` - -You should see the logs of both _Redis_ and _Scrolls_ crawling the chain from a remote relay node. If you're familiar with Redis CLI, you can run the following commands to see the data being cached: - -```sh -redis:6379> KEYS * -1) "c1.addr1qx0w02a2ez32tzh2wveu80nyml9hd50yp0udly07u5crl6x57nfgdzya4axrl8mfx450sxpyzskkl95sx5l7hcfw59psvu6ysx" -2) "c1.addr1qx68j552ywp6engr2s9xt7aawgpmr526krzt4mmzc8qe7p8qwjaawywglaawe74mwu726w49e8e0l9mexcwjk4kqm2tq5lmpd8" -3) "c1.addr1q90z7ujdyyct0jhcncrpv5ypzwytd3p7t0wv93anthmzvadjcq6ss65vaupzmy59dxj43lchlra0l482rh0qrw474snsgnq3df" -4) "c1.addr1w8vg4e5xdpad2jt0z775rt0alwmku3my2dmw8ezd884zvtssrq6fg" -5) "c1.addr1q9tj3tdhaxqyph568h7efh6h0f078m2pxyd0xgzq47htwe3vep55nfane06hggrc2gvnpdj4gcf26kzhkd3fs874hzhszja3lh" -6) "c1.addr1w8tqqyccvj7402zns2tea78d42etw520fzvf22zmyasjdtsv3e5rz" -redis:6379> SMEMBERS c1.addr1w8tqqyccvj7402zns2tea78d42etw520fzvf22zmyasjdtsv3e5rz -1) "2548228522837ea580bc55a3e6a09479deca499b5e7f3c08602a1f3191a178e7:20" -2) "04086c503512833c7a0c11fc85f7d0f0422db9d14b31275b3d4327c40c6fd73b:25" -redis:6379> -``` - -Once you're done with the testdive, you can clean your environment by running: - -```sh -docker-compose down -``` - ## FAQ ### Don't we have tools for this already? diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index ed473db0..0eaa9b94 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -6,7 +6,7 @@ - [From Source](./installation/from_source.md) - [Docker](./installation/docker.md) - [Usage](./usage/README.md) - - [Example Configuration](./installation/binary_release.md) + - [Example Configuration](./usage/example.md) - [Sources](./sources/README.md) - [Node-to-Node](./sources/n2n.md) - [Node-to-Client](./sources/n2c.md) @@ -15,5 +15,7 @@ - [Advanced Features](./advanced/README.md) - [Swarm Mode](./advanced/swarm_mode.md) - [Guides](./guides/README.md) - - [NodeJS](./guides/nodejs.md) - - [Python](./guides/python.md) + - [Testdrive](./guides/testdrive.md) + - [NodeJS Client](./guides/nodejs.md) + - [Python Client](./guides/python.md) + diff --git a/book/src/guides/nodejs.md b/book/src/guides/nodejs.md index f48fa8b1..d003153e 100644 --- a/book/src/guides/nodejs.md +++ b/book/src/guides/nodejs.md @@ -1,3 +1,3 @@ -# NodeJS +# NodeJS Client ## Read the data using NodeJS diff --git a/book/src/guides/python.md b/book/src/guides/python.md index c961114a..90903df8 100644 --- a/book/src/guides/python.md +++ b/book/src/guides/python.md @@ -1,4 +1,4 @@ -# Python +# Python Client ## Read the data using Python diff --git a/book/src/guides/testdrive.md b/book/src/guides/testdrive.md new file mode 100644 index 00000000..fbf3c1aa --- /dev/null +++ b/book/src/guides/testdrive.md @@ -0,0 +1,31 @@ +# Testdrive + +In the `testdrive` folder you'll find a minimal example that uses docker-compose to spin up a local Redis instance and a Scrolls daemon. You'll need Docker and docker-compose installed in your local machine. Run the following commands to start it: + +```sh +cd testdrive +docker-compose up +``` + +You should see the logs of both _Redis_ and _Scrolls_ crawling the chain from a remote relay node. If you're familiar with Redis CLI, you can run the following commands to see the data being cached: + +```sh +redis:6379> KEYS * +1) "c1.addr1qx0w02a2ez32tzh2wveu80nyml9hd50yp0udly07u5crl6x57nfgdzya4axrl8mfx450sxpyzskkl95sx5l7hcfw59psvu6ysx" +2) "c1.addr1qx68j552ywp6engr2s9xt7aawgpmr526krzt4mmzc8qe7p8qwjaawywglaawe74mwu726w49e8e0l9mexcwjk4kqm2tq5lmpd8" +3) "c1.addr1q90z7ujdyyct0jhcncrpv5ypzwytd3p7t0wv93anthmzvadjcq6ss65vaupzmy59dxj43lchlra0l482rh0qrw474snsgnq3df" +4) "c1.addr1w8vg4e5xdpad2jt0z775rt0alwmku3my2dmw8ezd884zvtssrq6fg" +5) "c1.addr1q9tj3tdhaxqyph568h7efh6h0f078m2pxyd0xgzq47htwe3vep55nfane06hggrc2gvnpdj4gcf26kzhkd3fs874hzhszja3lh" +6) "c1.addr1w8tqqyccvj7402zns2tea78d42etw520fzvf22zmyasjdtsv3e5rz" +redis:6379> SMEMBERS c1.addr1w8tqqyccvj7402zns2tea78d42etw520fzvf22zmyasjdtsv3e5rz +1) "2548228522837ea580bc55a3e6a09479deca499b5e7f3c08602a1f3191a178e7:20" +2) "04086c503512833c7a0c11fc85f7d0f0422db9d14b31275b3d4327c40c6fd73b:25" +redis:6379> +``` + +Once you're done with the testdive, you can clean your environment by running: + +```sh +docker-compose down +``` + From 0b94c4f0fc1c64dacb2649eafd614c02a92c2269 Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Sun, 19 Mar 2023 14:56:55 -0300 Subject: [PATCH 04/14] Wip docs --- book/src/SUMMARY.md | 3 +- book/src/guides/README.md | 2 + book/src/guides/nodejs.md | 2 - book/src/guides/python.md | 2 - book/src/guides/redis.md | 13 +++ book/src/installation/README.md | 3 + book/src/installation/binary_release.md | 5 +- book/src/installation/docker.md | 8 +- book/src/installation/from_source.md | 17 ++- book/src/introduction.md | 3 + book/src/reducers/README.md | 143 ++++++++++++++++++++++++ book/src/sources/README.md | 9 ++ book/src/storage/README.md | 9 ++ book/src/storage/redis.md | 5 + book/src/usage/README.md | 59 ++++++++++ book/src/usage/example.md | 42 ------- 16 files changed, 275 insertions(+), 50 deletions(-) create mode 100644 book/src/guides/redis.md create mode 100644 book/src/storage/redis.md delete mode 100644 book/src/usage/example.md diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 0eaa9b94..ba5894f9 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -6,16 +6,17 @@ - [From Source](./installation/from_source.md) - [Docker](./installation/docker.md) - [Usage](./usage/README.md) - - [Example Configuration](./usage/example.md) - [Sources](./sources/README.md) - [Node-to-Node](./sources/n2n.md) - [Node-to-Client](./sources/n2c.md) - [Reducers](./reducers/README.md) - [Storage](./storage/README.md) + - [Redis](./storage/redis.md) - [Advanced Features](./advanced/README.md) - [Swarm Mode](./advanced/swarm_mode.md) - [Guides](./guides/README.md) - [Testdrive](./guides/testdrive.md) - [NodeJS Client](./guides/nodejs.md) - [Python Client](./guides/python.md) + - [Redis-cli](./guides/redis.md) diff --git a/book/src/guides/README.md b/book/src/guides/README.md index 50a86734..59d46489 100644 --- a/book/src/guides/README.md +++ b/book/src/guides/README.md @@ -1 +1,3 @@ # Guides + +Quick informal guides to get starting using Scrlls with external tools. diff --git a/book/src/guides/nodejs.md b/book/src/guides/nodejs.md index d003153e..cd574bc4 100644 --- a/book/src/guides/nodejs.md +++ b/book/src/guides/nodejs.md @@ -1,3 +1 @@ # NodeJS Client - -## Read the data using NodeJS diff --git a/book/src/guides/python.md b/book/src/guides/python.md index 90903df8..0fb444f8 100644 --- a/book/src/guides/python.md +++ b/book/src/guides/python.md @@ -1,7 +1,5 @@ # Python Client -## Read the data using Python - Assuming you're using Redis as a storage backend (only one available ATM), we recommend using [redis-py](https://github.com/redis/redis-py) package to talk directly to the Redis instance. This is a very simple code snippet to query a the UTXOs by address. ```python diff --git a/book/src/guides/redis.md b/book/src/guides/redis.md new file mode 100644 index 00000000..1462f4e9 --- /dev/null +++ b/book/src/guides/redis.md @@ -0,0 +1,13 @@ +# Redis-cli + +If you are not familiar with Redis you might want to play with the [redis-cli](https://redis.io/docs/ui/cli/) command line application before start coding any serious application. This guide gives you the very basic commands needed to fetch information using `redis-cli`. + +See the [Redis](../storage/redis.md) (storage) section of the manual for Redis installation and configuration. + + +- Get all keys: KEYS * +- Get type of key: TYPE +- Get set members: SMEMBERS +- Get string: GET +- Clear database: FLUSHDB +- Clear all databases: FLUSHALL diff --git a/book/src/installation/README.md b/book/src/installation/README.md index 936c662c..18697015 100644 --- a/book/src/installation/README.md +++ b/book/src/installation/README.md @@ -5,3 +5,6 @@ Depending on your needs, Scrolls provides different installation options: - [Binary Release](binary_release.md): to use one of our pre-compiled binary releases for the supported platforms. - [From Source](from_source.md): to compile a binary from source code using Rust's toolchain - [Docker](docker.md): to run the tool from a pre-built docker image + + +You will also need a data storage backend installed in your system. Currently we support Redis, see our [Testdrive](../guides/testdrive.md) guide for a minimal docker image providing a Redis instance. diff --git a/book/src/installation/binary_release.md b/book/src/installation/binary_release.md index 035451a0..5b4c8c62 100644 --- a/book/src/installation/binary_release.md +++ b/book/src/installation/binary_release.md @@ -1,4 +1,7 @@ # Binary release +Installation using the pre-compiled binaries shared via [Github Releases](https://github.com/txpipe/scrolls/releases) -Installation using one of the pre-compiled binaries shared via [Github Releases](https://github.com/txpipe/scrolls/releases) +# Linux + +Binaries built with musl are statically linked. diff --git a/book/src/installation/docker.md b/book/src/installation/docker.md index f87417eb..8da22377 100644 --- a/book/src/installation/docker.md +++ b/book/src/installation/docker.md @@ -1,3 +1,9 @@ # Docker -Installation using the Docker image shared via [Github Packages](https://github.com/txpipe/scrolls/pkgs/container/scrolls) +Installation using the Docker image: + +``` +docker pull ghcr.io/txpipe/scrolls:v0.5.0 +``` + +Check [Github Packages](https://github.com/txpipe/scrolls/pkgs/container/scrolls) for newer versions. diff --git a/book/src/installation/from_source.md b/book/src/installation/from_source.md index d696eb04..c81061a7 100644 --- a/book/src/installation/from_source.md +++ b/book/src/installation/from_source.md @@ -1,6 +1,7 @@ # From Source -To compile from source, you'll need to have the Rust toolchain available in your development box. Execute the following command to clone and build the project: + +To compile from source, you'll need to have the Rust toolchain available in your development box. Execute the following commands to clone and build the project: ```sh git clone https://github.com/txpipe/scrolls.git @@ -8,3 +9,17 @@ cd scrolls cargo build ``` +Tell cargo to use git to fetch dependecies if it fails with `no authentication available` error: +``` +CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build +``` + +Execute scrolls with: + +``` +./target/debug/scrolls +``` + + + +TODO: `cargo install --path .` diff --git a/book/src/introduction.md b/book/src/introduction.md index 4f3b828c..4aeb6926 100644 --- a/book/src/introduction.md +++ b/book/src/introduction.md @@ -3,3 +3,6 @@ ## Motivation Since blockchain is a sequence of transactions it is far from trivial to get a fast read only access to data. Scrolls is a tool for building and maintaining read-optimized collections of Cardano's on-chain entities. It crawls the history of the chain and aggregates all data to reflect the current state of affairs. Once the whole history has been processed, Scrolls watches the tip of the chain to keep the collections up-to-date. + +## Under the hood +All the heavy lifting required to communicate with the Cardano node is done by the [Pallas](https://github.com/txpipe/pallas) library, which provides an implementation of the Ouroboros multiplexer and a few of the required mini-protocol state-machines (ChainSync and LocalState in particular). diff --git a/book/src/reducers/README.md b/book/src/reducers/README.md index 840b86d4..5d80e344 100644 --- a/book/src/reducers/README.md +++ b/book/src/reducers/README.md @@ -1 +1,144 @@ # Reducers + +## address_by_asset + +### Description +### Output Format + +
+
+
+ +## address_by_txo + +### Description +### Output Format + +
+
+
+ +## addresses_by_stake + +### Description +### Output Format + +
+
+
+ +## asset_holders_by_asset_id + +### Description +### Output Format + +
+
+
+ +## balance_by_address + +### Description +### Output Format + +
+
+
+ +## block_header_by_hash + +### Description +### Output Format + +
+
+
+ +## last_block_parameters + +### Description +### Output Format + +
+
+
+ +## point_by_tx + +### Description +### Output Format + +
+
+
+ +## pool_by_stake + +### Description +### Output Format + +
+
+
+ +## supply_by_asset + +### Description +### Output Format + +
+
+
+ +## tx_by_hash + +### Description +### Output Format + +
+
+
+ +## tx_count_by_address + +### Description +### Output Format + +
+
+
+ +## tx_count_by_native_token_policy_id + +### Description +### Output Format + +
+
+
+ +## utxo_by_address + +### Description +### Output Format + +
+
+
+ +## utxo_by_stake + +### Description +### Output Format + +
+
+
+## utxos_by_asset + +### Description +### Output Format + +
+
+
diff --git a/book/src/sources/README.md b/book/src/sources/README.md index e70306d4..c800a6f7 100644 --- a/book/src/sources/README.md +++ b/book/src/sources/README.md @@ -1 +1,10 @@ # Sources + +Sources represent the "origin" of the events processed by Scrolls. Any compatible source is responsible for feeding blockchain data into crolls's pipeline for further processing. + +## Built-in Sources + +These are the currently available sources included as part the main Scrolls codebase: + +- [N2N](n2n.md): an Ouroboros agent that connects to a Cardano node using node-2-node protocols +- [N2C](n2c.md): an Ouroboros agent that connects to a Cardano node using node-2-client protocols diff --git a/book/src/storage/README.md b/book/src/storage/README.md index e0e8ac41..cb1bae0a 100644 --- a/book/src/storage/README.md +++ b/book/src/storage/README.md @@ -1 +1,10 @@ # Storage + +Storage backends are "pluggable", any key-value storage mechanism is a potential candidate. Our backend of preference is Redis (and TBH, the only one implemented so far). It provides a very high "read" throughput, it can be shared across the network by multiple clients and can be used in cluster-mode for horizontal scaling. + + +## Available backends: + +- [Redis](./redis.md) + + diff --git a/book/src/storage/redis.md b/book/src/storage/redis.md new file mode 100644 index 00000000..bfb19a27 --- /dev/null +++ b/book/src/storage/redis.md @@ -0,0 +1,5 @@ +# Redis + +In case you don't have a redis instance running in your system, you might want to look at our [Testdrive](../guides/testdrive.md) guide for a minimal docker example providing a local redis instance. You can also look at the official Redis [Installation guide](https://redis.io/docs/getting-started/installation/). + +See our [Redis-cli basics](../guides/redis.md) guide to get started playing with Redis from the command line. diff --git a/book/src/usage/README.md b/book/src/usage/README.md index 8f04b05a..bc368eca 100644 --- a/book/src/usage/README.md +++ b/book/src/usage/README.md @@ -1 +1,60 @@ # Usage + +Once you have Scrolls and Redis installed in your system (see [Installation](../installation/index.md)) you can start the daemon with the following command: + +``` bash +/path/to/scrolls daemon --config /path/to/config.toml +``` + +If you have no experience using Redis, you can follow the [Redis-cli guide](../guides/redis.md) to learn the very basics needed to fetch data from redis using the command line application provided by Redis. For any real application you would need to use a client library in your language of preference. See our [Python](../guides/python.md) and [NodeJS](../guides/nodejs.md) guides. + +## Configuration +Scrolls daemon must be configured using a single `.toml` file. For the purpose of testing out Scrolls you can use the provided configuration located in `testdrive/simple/daemon.toml`. See below for an example config with examplantions. + +```toml +# get data from a relay node +[source] +type = "N2N" +address = "relays-new.cardano-mainnet.iohk.io:3001" + +# You can optionally enable enrichment (local db with transactions), this is needed for some reducers +[enrich] +type = "Sled" +db_path = "/opt/scrolls/sled_db" + +# enable the "UTXO by Address" collection +[[reducers]] +type = "UtxoByAddress" +# you can optionally prefix the keys in the collection +key_prefix = "c1" +# you can optionally only process UTXO from a set of predetermined addresses +filter = ["addr1qy8jecz3nal788f8t2zy6vj2l9ply3trpnkn2xuvv5rgu4m7y853av2nt8wc33agu3kuakvg0kaee0tfqhgelh2eeyyqgxmxw3"] + +# enable the "Point by Tx" collection +[[reducers]] +type = "PointByTx" +key_prefix = "c2" + +# store the collections in a local Redis +[storage] +type = "Redis" +connection_params = "redis://127.0.0.1:6379" + +# start reading from an arbitrary point in the chain +[intersect] +type = "Point" +value = [57867490, "c491c5006192de2c55a95fb3544f60b96bd1665accaf2dfa2ab12fc7191f016b"] + +# let Scrolls know that we're working with mainnet +[chain] +type = "Mainnet" +``` +### The source section +This section specifies the origin of the data. The special type field must always be present and containing a value matching any of the available built-in sources. The rest of the fields in the section will depend on the selected type. See the [Sources](../sources/index.md) section for a list of available options and their corresponding config values. +### The enrich section +### The reducers section +See the [Reducers](../reducers/index.md) section of the manual. +### The storage section +See the [Storage](../storage/index.md) section of the manual. +### The intersect section +### The chain section diff --git a/book/src/usage/example.md b/book/src/usage/example.md deleted file mode 100644 index 74e131df..00000000 --- a/book/src/usage/example.md +++ /dev/null @@ -1,42 +0,0 @@ -## Example Configuration - -This is an example configuration file: - -```toml -# get data from a relay node -[source] -type = "N2N" -address = "relays-new.cardano-mainnet.iohk.io:3001" - -# You can optionally enable enrichment (local db with transactions), this is needed for some reducers -[enrich] -type = "Sled" -db_path = "/opt/scrolls/sled_db" - -# enable the "UTXO by Address" collection -[[reducers]] -type = "UtxoByAddress" -# you can optionally prefix the keys in the collection -key_prefix = "c1" -# you can optionally only process UTXO from a set of predetermined addresses -filter = ["addr1qy8jecz3nal788f8t2zy6vj2l9ply3trpnkn2xuvv5rgu4m7y853av2nt8wc33agu3kuakvg0kaee0tfqhgelh2eeyyqgxmxw3"] - -# enable the "Point by Tx" collection -[[reducers]] -type = "PointByTx" -key_prefix = "c2" - -# store the collections in a local Redis -[storage] -type = "Redis" -connection_params = "redis://127.0.0.1:6379" - -# start reading from an arbitrary point in the chain -[intersect] -type = "Point" -value = [57867490, "c491c5006192de2c55a95fb3544f60b96bd1665accaf2dfa2ab12fc7191f016b"] - -# let Scrolls know that we're working with mainnet -[chain] -type = "Mainnet" -``` From 27ba57cc8a66c2be2ff41c20068d8b584c195776 Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Mon, 27 Mar 2023 14:41:36 -0300 Subject: [PATCH 05/14] Wip progress --- book/src/SUMMARY.md | 14 +-- book/src/configuration/README.md | 43 ++++++++ book/src/configuration/chain.md | 1 + book/src/configuration/enrich.md | 7 ++ book/src/configuration/intersect.md | 15 +++ book/src/configuration/policy.md | 1 + book/src/configuration/redis.md | 1 + book/src/configuration/reducers.md | 146 ++++++++++++++++++++++++++++ book/src/configuration/sources.md | 12 +++ book/src/configuration/storage.md | 12 +++ book/src/installation/docker.md | 4 +- book/src/reducers/README.md | 1 + book/src/sources/README.md | 10 -- book/src/sources/n2c.md | 1 - book/src/sources/n2n.md | 1 - book/src/usage/README.md | 52 +--------- 16 files changed, 251 insertions(+), 70 deletions(-) create mode 100644 book/src/configuration/README.md create mode 100644 book/src/configuration/chain.md create mode 100644 book/src/configuration/enrich.md create mode 100644 book/src/configuration/intersect.md create mode 100644 book/src/configuration/policy.md create mode 100644 book/src/configuration/redis.md create mode 100644 book/src/configuration/reducers.md create mode 100644 book/src/configuration/sources.md create mode 100644 book/src/configuration/storage.md delete mode 100644 book/src/sources/README.md delete mode 100644 book/src/sources/n2c.md delete mode 100644 book/src/sources/n2n.md diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index ba5894f9..c8d70aba 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -6,12 +6,14 @@ - [From Source](./installation/from_source.md) - [Docker](./installation/docker.md) - [Usage](./usage/README.md) -- [Sources](./sources/README.md) - - [Node-to-Node](./sources/n2n.md) - - [Node-to-Client](./sources/n2c.md) -- [Reducers](./reducers/README.md) -- [Storage](./storage/README.md) - - [Redis](./storage/redis.md) +- [Configuration](./configuration/README.md) + - [Sources](./configuration/sources.md) + - [Reducers](./configuration/reducers.md) + - [Storage](./configuration/storage.md) + - [Enrich](./configuration/enrich.md) + - [Intersect](./configuration/intersect.md) + - [Chain](./configuration/chain.md) + - [Policy](./configuration/policy.md) - [Advanced Features](./advanced/README.md) - [Swarm Mode](./advanced/swarm_mode.md) - [Guides](./guides/README.md) diff --git a/book/src/configuration/README.md b/book/src/configuration/README.md new file mode 100644 index 00000000..724e95d2 --- /dev/null +++ b/book/src/configuration/README.md @@ -0,0 +1,43 @@ +# Configuration + +Scrolls daemon must be configured using a single `.toml` file. For the purpose of testing out Scrolls you can use the provided configuration located in `testdrive/simple/daemon.toml`. See below for an example config with explantions. + +```toml +# get data from a relay node +[source] +type = "N2N" +address = "relays-new.cardano-mainnet.iohk.io:3001" + +# You can optionally enable enrichment (local db with transactions), this is needed for some reducers +[enrich] +type = "Sled" +db_path = "/opt/scrolls/sled_db" + +# enable the "UTXO by Address" collection +[[reducers]] +type = "UtxoByAddress" +# you can optionally prefix the keys in the collection +key_prefix = "c1" +# you can optionally only process UTXO from a set of predetermined addresses +filter = ["addr1qy8jecz3nal788f8t2zy6vj2l9ply3trpnkn2xuvv5rgu4m7y853av2nt8wc33agu3kuakvg0kaee0tfqhgelh2eeyyqgxmxw3"] + +# enable the "Point by Tx" collection +[[reducers]] +type = "PointByTx" +key_prefix = "c2" + +# store the collections in a local Redis +[storage] +type = "Redis" +connection_params = "redis://127.0.0.1:6379" + +# start reading from an arbitrary point in the chain +[intersect] +type = "Point" +value = [57867490, "c491c5006192de2c55a95fb3544f60b96bd1665accaf2dfa2ab12fc7191f016b"] + +# let Scrolls know that we're working with mainnet +[chain] +type = "Mainnet" +``` + diff --git a/book/src/configuration/chain.md b/book/src/configuration/chain.md new file mode 100644 index 00000000..867836c1 --- /dev/null +++ b/book/src/configuration/chain.md @@ -0,0 +1 @@ +# Chain diff --git a/book/src/configuration/enrich.md b/book/src/configuration/enrich.md new file mode 100644 index 00000000..99390f92 --- /dev/null +++ b/book/src/configuration/enrich.md @@ -0,0 +1,7 @@ +# Enrich + +## Fields +- type: `"Sled" | "Skip"` +- db_path **(*)**: `""` + +(*) Available only with `type = "Sled"` diff --git a/book/src/configuration/intersect.md b/book/src/configuration/intersect.md new file mode 100644 index 00000000..de73d639 --- /dev/null +++ b/book/src/configuration/intersect.md @@ -0,0 +1,15 @@ +# Intersect + +Scrolls provides 4 different strategies for finding the intersection point within the chain sync process. + +- `Origin`: Scrolls will start reading from the beginning of the chain. +- `Tip`: Scrolls will start reading from the current tip of the chain. +- `Point`: Scrolls will start reading from a particular point (slot, hash) in the chain. If the point is not found, the process will be terminated with a non-zero exit code. +- `Fallbacks`: Scrolls will start reading the first valid point within a set of alternative positions. If point is not valid, the process will fallback into the next available point in the list of options. If none of the points are valid, the process will be terminated with a non-zero exit code. + + +## Fields +- type: `"Tip" | "Origin" | "Point" | "Fallbacks"` +- value **(*)**: `(u64, String) | Vec<(u64, String)>` + +**(*)** Use `(u64, String)` with `type = "Point"` and `Vec<(u64, String)>` with `type = "Fallbacks"` diff --git a/book/src/configuration/policy.md b/book/src/configuration/policy.md new file mode 100644 index 00000000..e19eb184 --- /dev/null +++ b/book/src/configuration/policy.md @@ -0,0 +1 @@ +# Policy diff --git a/book/src/configuration/redis.md b/book/src/configuration/redis.md new file mode 100644 index 00000000..51a87de7 --- /dev/null +++ b/book/src/configuration/redis.md @@ -0,0 +1 @@ +# Redis diff --git a/book/src/configuration/reducers.md b/book/src/configuration/reducers.md new file mode 100644 index 00000000..eea5c04f --- /dev/null +++ b/book/src/configuration/reducers.md @@ -0,0 +1,146 @@ +# Reducers + +## address_by_asset + +### Description +### Output Format + +
+
+
+ +## address_by_txo + +### Description +### Output Format + +
+
+
+ +## addresses_by_stake + +### Description +### Output Format + +
+
+
+ +## asset_holders_by_asset_id + +### Description +### Output Format + +
+
+
+ +## balance_by_address + +### Description +### Output Format + +
+
+
+ +## block_header_by_hash + +### Description +### Output Format + +
+
+
+ +## last_block_parameters + +### Description +### Output Format + +
+
+
+ +## point_by_tx + +### Description +### Output Format + +
+
+
+ +## pool_by_stake + +### Description +### Output Format + +
+
+
+ +## supply_by_asset + +### Description +### Output Format + +
+
+
+ +## tx_by_hash + +### Description +### Output Format + +
+
+
+ +## tx_count_by_address + +### Description +### Output Format + +
+
+
+ +## tx_count_by_native_token_policy_id + +### Description +### Output Format + +
+
+
+ +## utxo_by_address + +### Description +### Output Format + +
+
+
+ +## utxo_by_stake + +### Description +### Output Format + +
+
+
+ +## utxos_by_asset + +### Description +### Output Format + +
+
+
+ diff --git a/book/src/configuration/sources.md b/book/src/configuration/sources.md new file mode 100644 index 00000000..46ddcef8 --- /dev/null +++ b/book/src/configuration/sources.md @@ -0,0 +1,12 @@ +# Sources + +Sources represent the "origin" of the events processed by Scrolls. Any compatible source is responsible for feeding blockchain data into crolls's pipeline for further processing. This section describes the currently available sources included as part the main Scrolls codebase. + +### Node-to-Node +The Node-to-Node (N2N) source uses Ouroboros mini-protocols to connect to a local or remote Cardano node through a tcp socket bearer and fetches block data using the ChainSync mini-protocol instantiated to "headers only" and the BlockFetch mini-protocol for retrieval of the actual block payload. + + +### Node-to-Client +The Node-to-Client (N2C) source uses Ouroboros mini-protocols to connect to a local Cardano node through a unix socket bearer and fetches block data using the ChainSync mini-protocol instantiated to "full blocks". + + diff --git a/book/src/configuration/storage.md b/book/src/configuration/storage.md new file mode 100644 index 00000000..9cd4abbc --- /dev/null +++ b/book/src/configuration/storage.md @@ -0,0 +1,12 @@ +# Storage + +Storage backends are "pluggable", any key-value storage mechanism is a potential candidate. Our backend of preference is Redis (and TBH, the only one implemented so far). It provides a very high "read" throughput, it can be shared across the network by multiple clients and can be used in cluster-mode for horizontal scaling. + + +## Available backends + +### Redis + +In case you don't have a redis instance running in your system, you might want to look at our [Testdrive](../guides/testdrive.md) guide for a minimal docker example providing a local redis instance. You can also look at the official Redis [Installation guide](https://redis.io/docs/getting-started/installation/). + +See our [Redis-cli basics](../guides/redis.md) guide to get started playing with Redis from the command line. diff --git a/book/src/installation/docker.md b/book/src/installation/docker.md index 8da22377..52d0a40d 100644 --- a/book/src/installation/docker.md +++ b/book/src/installation/docker.md @@ -1,9 +1,11 @@ # Docker -Installation using the Docker image: +Installation using the Docker image. Use the following command: ``` docker pull ghcr.io/txpipe/scrolls:v0.5.0 ``` Check [Github Packages](https://github.com/txpipe/scrolls/pkgs/container/scrolls) for newer versions. + +Check the [Testdrive](../guides/testdrive.md) guide for a minimal example that uses docker-compose to spin up a local Redis instance and a Scrolls daemon. diff --git a/book/src/reducers/README.md b/book/src/reducers/README.md index 5d80e344..82c286f0 100644 --- a/book/src/reducers/README.md +++ b/book/src/reducers/README.md @@ -134,6 +134,7 @@

+ ## utxos_by_asset ### Description diff --git a/book/src/sources/README.md b/book/src/sources/README.md deleted file mode 100644 index c800a6f7..00000000 --- a/book/src/sources/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Sources - -Sources represent the "origin" of the events processed by Scrolls. Any compatible source is responsible for feeding blockchain data into crolls's pipeline for further processing. - -## Built-in Sources - -These are the currently available sources included as part the main Scrolls codebase: - -- [N2N](n2n.md): an Ouroboros agent that connects to a Cardano node using node-2-node protocols -- [N2C](n2c.md): an Ouroboros agent that connects to a Cardano node using node-2-client protocols diff --git a/book/src/sources/n2c.md b/book/src/sources/n2c.md deleted file mode 100644 index f3e76af2..00000000 --- a/book/src/sources/n2c.md +++ /dev/null @@ -1 +0,0 @@ -# Node-to-Client diff --git a/book/src/sources/n2n.md b/book/src/sources/n2n.md deleted file mode 100644 index 51a2b802..00000000 --- a/book/src/sources/n2n.md +++ /dev/null @@ -1 +0,0 @@ -# Node-to-Node diff --git a/book/src/usage/README.md b/book/src/usage/README.md index bc368eca..3ff013bd 100644 --- a/book/src/usage/README.md +++ b/book/src/usage/README.md @@ -6,55 +6,5 @@ Once you have Scrolls and Redis installed in your system (see [Installation](../ /path/to/scrolls daemon --config /path/to/config.toml ``` -If you have no experience using Redis, you can follow the [Redis-cli guide](../guides/redis.md) to learn the very basics needed to fetch data from redis using the command line application provided by Redis. For any real application you would need to use a client library in your language of preference. See our [Python](../guides/python.md) and [NodeJS](../guides/nodejs.md) guides. +Check the [Configuration](../configuration/index.md) section of the manual for further information. If you have no experience using Redis, you can follow the [Redis-cli guide](../guides/redis.md) to learn the very basics needed to fetch data using the command line application provided by Redis. For any real application you would need to use a client library in your language of preference. See our [Python](../guides/python.md) and [NodeJS](../guides/nodejs.md) guides. -## Configuration -Scrolls daemon must be configured using a single `.toml` file. For the purpose of testing out Scrolls you can use the provided configuration located in `testdrive/simple/daemon.toml`. See below for an example config with examplantions. - -```toml -# get data from a relay node -[source] -type = "N2N" -address = "relays-new.cardano-mainnet.iohk.io:3001" - -# You can optionally enable enrichment (local db with transactions), this is needed for some reducers -[enrich] -type = "Sled" -db_path = "/opt/scrolls/sled_db" - -# enable the "UTXO by Address" collection -[[reducers]] -type = "UtxoByAddress" -# you can optionally prefix the keys in the collection -key_prefix = "c1" -# you can optionally only process UTXO from a set of predetermined addresses -filter = ["addr1qy8jecz3nal788f8t2zy6vj2l9ply3trpnkn2xuvv5rgu4m7y853av2nt8wc33agu3kuakvg0kaee0tfqhgelh2eeyyqgxmxw3"] - -# enable the "Point by Tx" collection -[[reducers]] -type = "PointByTx" -key_prefix = "c2" - -# store the collections in a local Redis -[storage] -type = "Redis" -connection_params = "redis://127.0.0.1:6379" - -# start reading from an arbitrary point in the chain -[intersect] -type = "Point" -value = [57867490, "c491c5006192de2c55a95fb3544f60b96bd1665accaf2dfa2ab12fc7191f016b"] - -# let Scrolls know that we're working with mainnet -[chain] -type = "Mainnet" -``` -### The source section -This section specifies the origin of the data. The special type field must always be present and containing a value matching any of the available built-in sources. The rest of the fields in the section will depend on the selected type. See the [Sources](../sources/index.md) section for a list of available options and their corresponding config values. -### The enrich section -### The reducers section -See the [Reducers](../reducers/index.md) section of the manual. -### The storage section -See the [Storage](../storage/index.md) section of the manual. -### The intersect section -### The chain section From aa72c84223fc82548ff3b3b94cc4ecfd6b2a80eb Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Mon, 27 Mar 2023 15:54:57 -0300 Subject: [PATCH 06/14] Wip documentation progress --- book/src/configuration/README.md | 2 +- book/src/configuration/chain.md | 26 ++++++++++++++++++++++++++ book/src/configuration/enrich.md | 12 ++++++++++-- book/src/configuration/intersect.md | 21 +++++++++++++++++++-- book/src/configuration/policy.md | 16 ++++++++++++++++ book/src/configuration/redis.md | 1 - book/src/configuration/sources.md | 25 +++++++++++++++++++++++-- book/src/configuration/storage.md | 18 ++++++++++++++---- 8 files changed, 109 insertions(+), 12 deletions(-) delete mode 100644 book/src/configuration/redis.md diff --git a/book/src/configuration/README.md b/book/src/configuration/README.md index 724e95d2..53cd9db8 100644 --- a/book/src/configuration/README.md +++ b/book/src/configuration/README.md @@ -1,6 +1,6 @@ # Configuration -Scrolls daemon must be configured using a single `.toml` file. For the purpose of testing out Scrolls you can use the provided configuration located in `testdrive/simple/daemon.toml`. See below for an example config with explantions. +Scrolls daemon must be configured using a single `.toml` file. For the purpose of testing out Scrolls you can use the provided configuration located in `testdrive/simple/daemon.toml`. See below for an example config with explantions and check the following sections of this books to understand in detail each section of the configuration file. ```toml # get data from a relay node diff --git a/book/src/configuration/chain.md b/book/src/configuration/chain.md index 867836c1..490f7c90 100644 --- a/book/src/configuration/chain.md +++ b/book/src/configuration/chain.md @@ -1 +1,27 @@ # Chain + +## Fields +- type: `"Mainnet" | "Testnet" | "PreProd" | "Preview" | "Custom"` +- magic (*): u64, +- byron_epoch_length (*): u32, +- byron_slot_length (*): u32, +- byron_known_slot (*): u64, +- byron_known_hash (*): String, +- byron_known_time (*): u64, +- shelley_epoch_length (*): u32, +- shelley_slot_length (*): u32, +- shelley_known_slot (*): u64, +- shelley_known_hash (*): String, +- shelley_known_time (*): u64, +- address_network_id (*): u8, +- adahandle_policy (*): String, + + +(*) Use only with `type = "Custom"` + +## Example + +``` toml +[chain] +type = "Mainnet" +``` diff --git a/book/src/configuration/enrich.md b/book/src/configuration/enrich.md index 99390f92..495e0256 100644 --- a/book/src/configuration/enrich.md +++ b/book/src/configuration/enrich.md @@ -2,6 +2,14 @@ ## Fields - type: `"Sled" | "Skip"` -- db_path **(*)**: `""` +- db_path (*): `""` -(*) Available only with `type = "Sled"` +(*) Use only with `type = "Sled"` + +## Example + +``` toml +[enrich] +type = "Sled" +db_path = "/opt/scrolls/sled_db" +``` diff --git a/book/src/configuration/intersect.md b/book/src/configuration/intersect.md index de73d639..249fd964 100644 --- a/book/src/configuration/intersect.md +++ b/book/src/configuration/intersect.md @@ -10,6 +10,23 @@ Scrolls provides 4 different strategies for finding the intersection point withi ## Fields - type: `"Tip" | "Origin" | "Point" | "Fallbacks"` -- value **(*)**: `(u64, String) | Vec<(u64, String)>` +- value (*): `(u64, String) | Vec<(u64, String)>` -**(*)** Use `(u64, String)` with `type = "Point"` and `Vec<(u64, String)>` with `type = "Fallbacks"` +(*) Use value of type `(u64, String)` with `type = "Point"` and value of type `Vec<(u64, String)>` with `type = "Fallbacks"` + +## Examples + +``` toml +[intersect] +type = "Point" +value = [57867490, "c491c5006192de2c55a95fb3544f60b96bd1665accaf2dfa2ab12fc7191f016b"] +``` + +``` toml +[intersect] +type = "Fallbacks" +value = [ + [12345678, "this_is_not_a_valid_hash_ff1b93cdfd997d4ea93e7d930908aa5905d788f"], + [57867490, "c491c5006192de2c55a95fb3544f60b96bd1665accaf2dfa2ab12fc7191f016b"] + ] +``` diff --git a/book/src/configuration/policy.md b/book/src/configuration/policy.md index e19eb184..7b81d06b 100644 --- a/book/src/configuration/policy.md +++ b/book/src/configuration/policy.md @@ -1 +1,17 @@ # Policy + +## Fields +- missing_data: `"Skip" | "Warn" | "Default"` +- cbor_errors: `"Skip" | "Warn" | "Default"` +- ledger_errors: `"Skip" | "Warn" | "Default"` +- any_error: `"Skip" | "Warn" | "Default"` + + +## Example + +``` toml +[policy] +cbor_errors = "Skip" +missing_data = "Warn" +``` + diff --git a/book/src/configuration/redis.md b/book/src/configuration/redis.md deleted file mode 100644 index 51a87de7..00000000 --- a/book/src/configuration/redis.md +++ /dev/null @@ -1 +0,0 @@ -# Redis diff --git a/book/src/configuration/sources.md b/book/src/configuration/sources.md index 46ddcef8..b39850d2 100644 --- a/book/src/configuration/sources.md +++ b/book/src/configuration/sources.md @@ -2,11 +2,32 @@ Sources represent the "origin" of the events processed by Scrolls. Any compatible source is responsible for feeding blockchain data into crolls's pipeline for further processing. This section describes the currently available sources included as part the main Scrolls codebase. -### Node-to-Node +## Node-to-Node The Node-to-Node (N2N) source uses Ouroboros mini-protocols to connect to a local or remote Cardano node through a tcp socket bearer and fetches block data using the ChainSync mini-protocol instantiated to "headers only" and the BlockFetch mini-protocol for retrieval of the actual block payload. -### Node-to-Client +## Node-to-Client The Node-to-Client (N2C) source uses Ouroboros mini-protocols to connect to a local Cardano node through a unix socket bearer and fetches block data using the ChainSync mini-protocol instantiated to "full blocks". +## Fields +- type: `"N2N" | "N2C"` +- address (*): `""` +- path (**): `""` + +(*) Use only with `type = "N2N"` + +(**) Use only with `type = "N2C` + +## Examples +``` toml +[source] +type = "N2N" +address = "relays-new.cardano-mainnet.iohk.io:3001" +``` + +``` toml +[source] +type = "N2C" +path = "data/cardano-node-mainnet/node.socket" +``` diff --git a/book/src/configuration/storage.md b/book/src/configuration/storage.md index 9cd4abbc..30c895ac 100644 --- a/book/src/configuration/storage.md +++ b/book/src/configuration/storage.md @@ -2,11 +2,21 @@ Storage backends are "pluggable", any key-value storage mechanism is a potential candidate. Our backend of preference is Redis (and TBH, the only one implemented so far). It provides a very high "read" throughput, it can be shared across the network by multiple clients and can be used in cluster-mode for horizontal scaling. +In case you don't have a redis instance running in your system, you might want to look at our [Testdrive](../guides/testdrive.md) guide for a minimal docker example providing a local redis instance. You can also look at the official Redis [Installation guide](https://redis.io/docs/getting-started/installation/). -## Available backends +See our [Redis-cli basics](../guides/redis.md) guide to get started playing with Redis from the command line. -### Redis +### Fields -In case you don't have a redis instance running in your system, you might want to look at our [Testdrive](../guides/testdrive.md) guide for a minimal docker example providing a local redis instance. You can also look at the official Redis [Installation guide](https://redis.io/docs/getting-started/installation/). +- type: `"Redis" | "Skip"` +- connection_params (*): `""` -See our [Redis-cli basics](../guides/redis.md) guide to get started playing with Redis from the command line. +(*) Use only with `type = "Redis"` + +#### Example + +``` toml +[storage] +type = "Redis" +connection_params = "redis://127.0.0.1:6379" +``` From 97a0514f5ce8799d71749a222ede1493773d1e71 Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Mon, 27 Mar 2023 16:00:28 -0300 Subject: [PATCH 07/14] Minor doc improvements --- book/src/configuration/intersect.md | 2 ++ book/src/configuration/sources.md | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/book/src/configuration/intersect.md b/book/src/configuration/intersect.md index 249fd964..51dbdb7b 100644 --- a/book/src/configuration/intersect.md +++ b/book/src/configuration/intersect.md @@ -16,12 +16,14 @@ Scrolls provides 4 different strategies for finding the intersection point withi ## Examples +Using **Point**: ``` toml [intersect] type = "Point" value = [57867490, "c491c5006192de2c55a95fb3544f60b96bd1665accaf2dfa2ab12fc7191f016b"] ``` +Using **Fallbacks**: ``` toml [intersect] type = "Fallbacks" diff --git a/book/src/configuration/sources.md b/book/src/configuration/sources.md index b39850d2..24cbbbe5 100644 --- a/book/src/configuration/sources.md +++ b/book/src/configuration/sources.md @@ -2,12 +2,9 @@ Sources represent the "origin" of the events processed by Scrolls. Any compatible source is responsible for feeding blockchain data into crolls's pipeline for further processing. This section describes the currently available sources included as part the main Scrolls codebase. -## Node-to-Node -The Node-to-Node (N2N) source uses Ouroboros mini-protocols to connect to a local or remote Cardano node through a tcp socket bearer and fetches block data using the ChainSync mini-protocol instantiated to "headers only" and the BlockFetch mini-protocol for retrieval of the actual block payload. +The **Node-to-Node (N2N)** source uses Ouroboros mini-protocols to connect to a local or remote Cardano node through a tcp socket bearer and fetches block data using the ChainSync mini-protocol instantiated to "headers only" and the BlockFetch mini-protocol for retrieval of the actual block payload. - -## Node-to-Client -The Node-to-Client (N2C) source uses Ouroboros mini-protocols to connect to a local Cardano node through a unix socket bearer and fetches block data using the ChainSync mini-protocol instantiated to "full blocks". +The **Node-to-Client (N2C)** source uses Ouroboros mini-protocols to connect to a local Cardano node through a unix socket bearer and fetches block data using the ChainSync mini-protocol instantiated to "full blocks". ## Fields @@ -29,5 +26,5 @@ address = "relays-new.cardano-mainnet.iohk.io:3001" ``` toml [source] type = "N2C" -path = "data/cardano-node-mainnet/node.socket" +path = "/path/to/node.socket" ``` From 4ec929213b5ce6845d414ecef20929757674c600 Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Mon, 27 Mar 2023 16:05:50 -0300 Subject: [PATCH 08/14] Delete unused files --- book/src/guides/redis.md | 15 ++-- book/src/reducers/README.md | 145 ------------------------------------ book/src/storage/README.md | 10 --- book/src/storage/redis.md | 5 -- 4 files changed, 6 insertions(+), 169 deletions(-) delete mode 100644 book/src/reducers/README.md delete mode 100644 book/src/storage/README.md delete mode 100644 book/src/storage/redis.md diff --git a/book/src/guides/redis.md b/book/src/guides/redis.md index 1462f4e9..ba047c6e 100644 --- a/book/src/guides/redis.md +++ b/book/src/guides/redis.md @@ -2,12 +2,9 @@ If you are not familiar with Redis you might want to play with the [redis-cli](https://redis.io/docs/ui/cli/) command line application before start coding any serious application. This guide gives you the very basic commands needed to fetch information using `redis-cli`. -See the [Redis](../storage/redis.md) (storage) section of the manual for Redis installation and configuration. - - -- Get all keys: KEYS * -- Get type of key: TYPE -- Get set members: SMEMBERS -- Get string: GET -- Clear database: FLUSHDB -- Clear all databases: FLUSHALL +- Get all keys: `KEYS *` +- Get type of key: `TYPE` +- Get set members: `SMEMBERS` +- Get string: `GET` +- Clear database: `FLUSHDB` +- Clear all databases: `FLUSHALL` diff --git a/book/src/reducers/README.md b/book/src/reducers/README.md deleted file mode 100644 index 82c286f0..00000000 --- a/book/src/reducers/README.md +++ /dev/null @@ -1,145 +0,0 @@ -# Reducers - -## address_by_asset - -### Description -### Output Format - -
-
-
- -## address_by_txo - -### Description -### Output Format - -
-
-
- -## addresses_by_stake - -### Description -### Output Format - -
-
-
- -## asset_holders_by_asset_id - -### Description -### Output Format - -
-
-
- -## balance_by_address - -### Description -### Output Format - -
-
-
- -## block_header_by_hash - -### Description -### Output Format - -
-
-
- -## last_block_parameters - -### Description -### Output Format - -
-
-
- -## point_by_tx - -### Description -### Output Format - -
-
-
- -## pool_by_stake - -### Description -### Output Format - -
-
-
- -## supply_by_asset - -### Description -### Output Format - -
-
-
- -## tx_by_hash - -### Description -### Output Format - -
-
-
- -## tx_count_by_address - -### Description -### Output Format - -
-
-
- -## tx_count_by_native_token_policy_id - -### Description -### Output Format - -
-
-
- -## utxo_by_address - -### Description -### Output Format - -
-
-
- -## utxo_by_stake - -### Description -### Output Format - -
-
-
- -## utxos_by_asset - -### Description -### Output Format - -
-
-
diff --git a/book/src/storage/README.md b/book/src/storage/README.md deleted file mode 100644 index cb1bae0a..00000000 --- a/book/src/storage/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Storage - -Storage backends are "pluggable", any key-value storage mechanism is a potential candidate. Our backend of preference is Redis (and TBH, the only one implemented so far). It provides a very high "read" throughput, it can be shared across the network by multiple clients and can be used in cluster-mode for horizontal scaling. - - -## Available backends: - -- [Redis](./redis.md) - - diff --git a/book/src/storage/redis.md b/book/src/storage/redis.md deleted file mode 100644 index bfb19a27..00000000 --- a/book/src/storage/redis.md +++ /dev/null @@ -1,5 +0,0 @@ -# Redis - -In case you don't have a redis instance running in your system, you might want to look at our [Testdrive](../guides/testdrive.md) guide for a minimal docker example providing a local redis instance. You can also look at the official Redis [Installation guide](https://redis.io/docs/getting-started/installation/). - -See our [Redis-cli basics](../guides/redis.md) guide to get started playing with Redis from the command line. From d1cc7e9f357e5820fd63bd7d10cfb56a3256a997 Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Mon, 27 Mar 2023 19:18:27 -0300 Subject: [PATCH 09/14] Wip doc progress --- book/src/configuration/README.md | 2 +- book/src/guides/README.md | 2 +- book/src/guides/redis.md | 2 +- book/src/usage/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/book/src/configuration/README.md b/book/src/configuration/README.md index 53cd9db8..1cba270a 100644 --- a/book/src/configuration/README.md +++ b/book/src/configuration/README.md @@ -1,6 +1,6 @@ # Configuration -Scrolls daemon must be configured using a single `.toml` file. For the purpose of testing out Scrolls you can use the provided configuration located in `testdrive/simple/daemon.toml`. See below for an example config with explantions and check the following sections of this books to understand in detail each section of the configuration file. +Scrolls daemon must be configured using a single `.toml` file. For the purpose of testing out Scrolls you can use the provided configuration located in `testdrive/simple/daemon.toml`. See below for an example config with explantions and check the following sections of this book to understand in detail each section of the configuration file. ```toml # get data from a relay node diff --git a/book/src/guides/README.md b/book/src/guides/README.md index 59d46489..c2fe4136 100644 --- a/book/src/guides/README.md +++ b/book/src/guides/README.md @@ -1,3 +1,3 @@ # Guides -Quick informal guides to get starting using Scrlls with external tools. +Quick informal guides to get starting using Scrolls with external tools. diff --git a/book/src/guides/redis.md b/book/src/guides/redis.md index ba047c6e..181a3711 100644 --- a/book/src/guides/redis.md +++ b/book/src/guides/redis.md @@ -1,6 +1,6 @@ # Redis-cli -If you are not familiar with Redis you might want to play with the [redis-cli](https://redis.io/docs/ui/cli/) command line application before start coding any serious application. This guide gives you the very basic commands needed to fetch information using `redis-cli`. +If you are not familiar with Redis you might want to play with the [redis-cli](https://redis.io/docs/ui/cli/) command line application before start coding any serious application. This guide gives you the very basic `redis-cli` commands needed to fetch information from redis databases. - Get all keys: `KEYS *` - Get type of key: `TYPE` diff --git a/book/src/usage/README.md b/book/src/usage/README.md index 3ff013bd..d937ec79 100644 --- a/book/src/usage/README.md +++ b/book/src/usage/README.md @@ -6,5 +6,5 @@ Once you have Scrolls and Redis installed in your system (see [Installation](../ /path/to/scrolls daemon --config /path/to/config.toml ``` -Check the [Configuration](../configuration/index.md) section of the manual for further information. If you have no experience using Redis, you can follow the [Redis-cli guide](../guides/redis.md) to learn the very basics needed to fetch data using the command line application provided by Redis. For any real application you would need to use a client library in your language of preference. See our [Python](../guides/python.md) and [NodeJS](../guides/nodejs.md) guides. +Check the [Configuration](../configuration/index.md) section of the manual for further information. If you have no experience using Redis, you can follow the [Redis-cli guide](../guides/redis.md) to learn the basic commands needed to fetch data using the command line application provided by Redis. For any real application you would need to use a client library in your language of preference. See our [Python](../guides/python.md) and [NodeJS](../guides/nodejs.md) guides. From 2bf505ac58950a09d3e3b552551bdd3f946a38ea Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Wed, 29 Mar 2023 11:02:31 -0300 Subject: [PATCH 10/14] Wip doc progress --- book/src/configuration/README.md | 2 +- book/src/configuration/chain.md | 61 +++++++++++- book/src/configuration/enrich.md | 1 + book/src/configuration/reducers.md | 143 ++++++++++++++++++++++++----- book/src/usage/README.md | 2 +- 5 files changed, 185 insertions(+), 24 deletions(-) diff --git a/book/src/configuration/README.md b/book/src/configuration/README.md index 1cba270a..f1b576c2 100644 --- a/book/src/configuration/README.md +++ b/book/src/configuration/README.md @@ -1,6 +1,6 @@ # Configuration -Scrolls daemon must be configured using a single `.toml` file. For the purpose of testing out Scrolls you can use the provided configuration located in `testdrive/simple/daemon.toml`. See below for an example config with explantions and check the following sections of this book to understand in detail each section of the configuration file. +Scrolls daemon must be configured using a single `.toml` file. For the purpose of testing out Scrolls you can use the provided configuration located in `testdrive/simple/daemon.toml`. See below for another example config with explantions and check the following sections of this book to understand in detail each section of the configuration file. ```toml # get data from a relay node diff --git a/book/src/configuration/chain.md b/book/src/configuration/chain.md index 490f7c90..7fae1b4a 100644 --- a/book/src/configuration/chain.md +++ b/book/src/configuration/chain.md @@ -1,5 +1,7 @@ # Chain +Specify which network to fetch data from. + ## Fields - type: `"Mainnet" | "Testnet" | "PreProd" | "Preview" | "Custom"` - magic (*): u64, @@ -19,9 +21,66 @@ (*) Use only with `type = "Custom"` -## Example +## Examples +Using mainnet ``` toml [chain] type = "Mainnet" ``` + +Using mainnet (custom values): +``` toml +[chain] +type = "Custom" +magic = 764824073 +byron_epoch_length = 432000 +byron_slot_length = 20 +byron_known_slot = 0 +byron_known_time = 1506203091 +byron_known_hash = "f0f7892b5c333cffc4b3c4344de48af4cc63f55e44936196f365a9ef2244134f" +shelley_epoch_length = 432000 +shelley_slot_length = 1 +shelley_known_slot = 4492800 +shelley_known_hash = "aa83acbf5904c0edfe4d79b3689d3d00fcfc553cf360fd2229b98d464c28e9de" +shelley_known_time = 1596059091 +address_network_id = 1 +adahandle_policy = "f0ff48bbb7bbe9d59a40f1ce90e9e9d0ff5002ec48f232b49ca0fb9a" + +``` + +Using preview (custom values): +``` toml +magic = 2 +byron_epoch_length = 432000 +byron_slot_length = 20 +byron_known_slot = 0 +byron_known_hash = "" +byron_known_time = 1660003200 +shelley_epoch_length = 432000 +shelley_slot_length = 1 +shelley_known_slot = 25260 +shelley_known_hash = "cac921895ef5f2e85f7e6e6b51b663ab81b3605cd47d6b6d66e8e785e5c65011" +shelley_known_time = 1660003200 +address_network_id = 0 +adahandle_policy = "" +``` + +Using preprod (custom values): +``` toml +[chain] +type = "Custom" +magic = 1 +byron_epoch_length = 432000 +byron_slot_length = 20 +byron_known_slot = 0 +byron_known_hash = "9ad7ff320c9cf74e0f5ee78d22a85ce42bb0a487d0506bf60cfb5a91ea4497d2" +byron_known_time = 1654041600 +shelley_epoch_length = 432000 +shelley_slot_length = 1 +shelley_known_slot = 86400 +shelley_known_hash = "c4a1595c5cc7a31eda9e544986fe9387af4e3491afe0ca9a80714f01951bbd5c" +shelley_known_time = 1654041600 +address_network_id = 0 +adahandle_policy = "" +``` diff --git a/book/src/configuration/enrich.md b/book/src/configuration/enrich.md index 495e0256..a0ed99a9 100644 --- a/book/src/configuration/enrich.md +++ b/book/src/configuration/enrich.md @@ -1,4 +1,5 @@ # Enrich +Store UTXOs information in a local DB, this is needed for some reducers to work. Currently, only [Sled](https://github.com/spacejam/sled) supported. ## Fields - type: `"Sled" | "Skip"` diff --git a/book/src/configuration/reducers.md b/book/src/configuration/reducers.md index eea5c04f..552071bf 100644 --- a/book/src/configuration/reducers.md +++ b/book/src/configuration/reducers.md @@ -1,8 +1,41 @@ # Reducers + +- [address_by_asset](#address_by_asset) +- [address_by_txo](#address_by_txo) +- [addresses_by_stake](#addresses_by_stake) +- [asset_holders_by_asset_id](#asset_holders_by_asset_id) +- [balance_by_address](#balance_by_address) +- [block_header_by_hash](#block_header_by_hash) +- [last_block_parameters](#last_block_parameters) +- [point_by_tx](#point_by_tx) +- [pool_by_stake](#pool_by_stake) +- [supply_by_asset](#supply_by_asset) +- [tx_by_hash](#tx_by_hash) +- [tx_count_by_address](#tx_count_by_address) +- [tx_count_by_native_token_policy_id](#tx_count_by_native_token_policy_id) +- [utxo_by_address](#utxo_by_address) +- [utxo_by_stake](#utxo_by_stake) +- [utxos_by_asset](#utxos_by_asset) + + +
+
+
+ ## address_by_asset -### Description +### Config + +- key_prefix: `Option` +- filter: `Option>` +- policy_id_hex: `String` +- convert_to_ascii(*): `Option` + +(*) default is true. + +### Example + ### Output Format
@@ -11,7 +44,12 @@ ## address_by_txo -### Description +### Config + +- key_prefix: `Option` +- filter: `Option` + +### Example ### Output Format
@@ -20,7 +58,12 @@ ## addresses_by_stake -### Description +### Config + +- key_prefix: `Option` +- filter: `Option>` + +### Example ### Output Format
@@ -29,7 +72,18 @@ ## asset_holders_by_asset_id -### Description +### Config + +- key_prefix: `Option` +- filter: `Option` +- aggr_by: `Option` + + /// Policies to match + /// + /// If specified only those policy ids as hex will be taken into account, if + /// not all policy ids will be indexed. +- policy_ids_hex: `Option>` +### Example ### Output Format
@@ -38,7 +92,10 @@ ## balance_by_address -### Description +### Config +- key_prefix: `Option` +- filter: `Option` +### Example ### Output Format
@@ -47,7 +104,11 @@ ## block_header_by_hash -### Description +### Config +- key_prefix: `Option` +- filter: `Option` + +### Example ### Output Format
@@ -56,7 +117,10 @@ ## last_block_parameters -### Description +### Config +- key_prefix: `Option` + +### Example ### Output Format
@@ -65,7 +129,10 @@ ## point_by_tx -### Description +### Config +- key_prefix: `Option` + +### Example ### Output Format
@@ -74,7 +141,10 @@ ## pool_by_stake -### Description +### Config +- key_prefix: `Option` + +### Example ### Output Format
@@ -83,7 +153,11 @@ ## supply_by_asset -### Description +### Config +- key_prefix: `Option` +- policy_ids_hex: `Option>` + +### Example ### Output Format
@@ -92,7 +166,19 @@ ## tx_by_hash -### Description +### Config +- key_prefix: `Option` +- filter: `Option` +- projection: `"Cbor" | "Json"` + +### Example + +``` toml +[[reducers]] +type: "TxByHash" +key_prefix: "c1" +``` + ### Output Format
@@ -101,7 +187,11 @@ ## tx_count_by_address -### Description +### Config +- key_prefix: `Option` +- filter: `Option` + +### Example ### Output Format
@@ -110,7 +200,11 @@ ## tx_count_by_native_token_policy_id -### Description +### Config +- key_prefix: `Option` +- aggr_by: `Option` + +### Example ### Output Format
@@ -119,7 +213,11 @@ ## utxo_by_address -### Description +### Config +- key_prefix: `Option` +- filter: `Option>` + +### Example ### Output Format
@@ -128,7 +226,11 @@ ## utxo_by_stake -### Description +### Config +- key_prefix: `Option` +- filter: `Option>` + +### Example ### Output Format
@@ -137,10 +239,9 @@ ## utxos_by_asset -### Description -### Output Format - -
-
-
+### Config +- key_prefix: `Option` +- policy_ids_hex: `Option>` +### Example +### Output Format diff --git a/book/src/usage/README.md b/book/src/usage/README.md index d937ec79..6e856fea 100644 --- a/book/src/usage/README.md +++ b/book/src/usage/README.md @@ -6,5 +6,5 @@ Once you have Scrolls and Redis installed in your system (see [Installation](../ /path/to/scrolls daemon --config /path/to/config.toml ``` -Check the [Configuration](../configuration/index.md) section of the manual for further information. If you have no experience using Redis, you can follow the [Redis-cli guide](../guides/redis.md) to learn the basic commands needed to fetch data using the command line application provided by Redis. For any real application you would need to use a client library in your language of preference. See our [Python](../guides/python.md) and [NodeJS](../guides/nodejs.md) guides. +Check the [Configuration](../configuration/index.md) section of the manual for further information. If you have no experience using Redis, you can follow the [Redis-cli guide](../guides/redis.md) to learn the basic commands needed to query data using the command line application provided by Redis. For any real application you would need to use a client library in your language of preference. See our [Python](../guides/python.md) and [NodeJS](../guides/nodejs.md) guides. From d185c3aecb7dc31d6e57f042a473f47b196d13a4 Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Wed, 29 Mar 2023 14:50:49 -0300 Subject: [PATCH 11/14] Wip documentation progress --- book/src/configuration/README.md | 1 + book/src/configuration/chain.md | 64 +++++++----------------------- book/src/configuration/enrich.md | 2 +- book/src/configuration/reducers.md | 39 ++++++++++++++++++ book/src/configuration/sources.md | 5 ++- book/src/configuration/storage.md | 3 +- 6 files changed, 60 insertions(+), 54 deletions(-) diff --git a/book/src/configuration/README.md b/book/src/configuration/README.md index f1b576c2..9f5ecb0b 100644 --- a/book/src/configuration/README.md +++ b/book/src/configuration/README.md @@ -41,3 +41,4 @@ value = [57867490, "c491c5006192de2c55a95fb3544f60b96bd1665accaf2dfa2ab12fc7191f type = "Mainnet" ``` +Sometimes it helps to use [toml2json](https://github.com/woodruffw/toml2json) and [jq](https://stedolan.github.io/jq/) if you need to visualize toml files as json. Example: `toml2json ./configuration.toml | jq` diff --git a/book/src/configuration/chain.md b/book/src/configuration/chain.md index 7fae1b4a..b4f9a659 100644 --- a/book/src/configuration/chain.md +++ b/book/src/configuration/chain.md @@ -4,19 +4,19 @@ Specify which network to fetch data from. ## Fields - type: `"Mainnet" | "Testnet" | "PreProd" | "Preview" | "Custom"` -- magic (*): u64, -- byron_epoch_length (*): u32, -- byron_slot_length (*): u32, -- byron_known_slot (*): u64, -- byron_known_hash (*): String, -- byron_known_time (*): u64, -- shelley_epoch_length (*): u32, -- shelley_slot_length (*): u32, -- shelley_known_slot (*): u64, -- shelley_known_hash (*): String, -- shelley_known_time (*): u64, -- address_network_id (*): u8, -- adahandle_policy (*): String, +- magic (*): `u64`, +- byron_epoch_length (*): `u32`, +- byron_slot_length (*): `u32`, +- byron_known_slot (*): `u64`, +- byron_known_hash (*): `String`, +- byron_known_time (*): `u64`, +- shelley_epoch_length (*): `u32`, +- shelley_slot_length (*): `u32`, +- shelley_known_slot (*): `u64`, +- shelley_known_hash (*): `String`, +- shelley_known_time (*): `u64`, +- address_network_id (*): `u8`, +- adahandle_policy (*): `String`, (*) Use only with `type = "Custom"` @@ -29,7 +29,7 @@ Using mainnet type = "Mainnet" ``` -Using mainnet (custom values): +Using custom values (mainnet): ``` toml [chain] type = "Custom" @@ -48,39 +48,3 @@ address_network_id = 1 adahandle_policy = "f0ff48bbb7bbe9d59a40f1ce90e9e9d0ff5002ec48f232b49ca0fb9a" ``` - -Using preview (custom values): -``` toml -magic = 2 -byron_epoch_length = 432000 -byron_slot_length = 20 -byron_known_slot = 0 -byron_known_hash = "" -byron_known_time = 1660003200 -shelley_epoch_length = 432000 -shelley_slot_length = 1 -shelley_known_slot = 25260 -shelley_known_hash = "cac921895ef5f2e85f7e6e6b51b663ab81b3605cd47d6b6d66e8e785e5c65011" -shelley_known_time = 1660003200 -address_network_id = 0 -adahandle_policy = "" -``` - -Using preprod (custom values): -``` toml -[chain] -type = "Custom" -magic = 1 -byron_epoch_length = 432000 -byron_slot_length = 20 -byron_known_slot = 0 -byron_known_hash = "9ad7ff320c9cf74e0f5ee78d22a85ce42bb0a487d0506bf60cfb5a91ea4497d2" -byron_known_time = 1654041600 -shelley_epoch_length = 432000 -shelley_slot_length = 1 -shelley_known_slot = 86400 -shelley_known_hash = "c4a1595c5cc7a31eda9e544986fe9387af4e3491afe0ca9a80714f01951bbd5c" -shelley_known_time = 1654041600 -address_network_id = 0 -adahandle_policy = "" -``` diff --git a/book/src/configuration/enrich.md b/book/src/configuration/enrich.md index a0ed99a9..5189ba18 100644 --- a/book/src/configuration/enrich.md +++ b/book/src/configuration/enrich.md @@ -3,7 +3,7 @@ Store UTXOs information in a local DB, this is needed for some reducers to work. ## Fields - type: `"Sled" | "Skip"` -- db_path (*): `""` +- db_path (*): `String` (*) Use only with `type = "Sled"` diff --git a/book/src/configuration/reducers.md b/book/src/configuration/reducers.md index 552071bf..309ec5d3 100644 --- a/book/src/configuration/reducers.md +++ b/book/src/configuration/reducers.md @@ -19,6 +19,45 @@ - [utxos_by_asset](#utxos_by_asset) +## Predicates + +Predicates can be used as filters by many reducers. + +``` +pub enum Predicate { + AllOf(Vec), + AnyOf(Vec), + Not(Box), + Block(BlockPattern), + Transaction(TransactionPattern), + InputAddress(AddressPattern), + OutputAddress(AddressPattern), + WithdrawalAddress(AddressPattern), + CollateralAddress(AddressPattern), + Address(AddressPattern), +} +``` + +``` +pub struct BlockPattern { + pub slot_before: Option, + pub slot_after: Option, +} +``` + +``` +#[derive(Deserialize, Clone, Default)] +pub struct AddressPattern { + pub exact_hex: Option, + pub exact_bech32: Option, + pub payment_hex: Option, + pub payment_bech32: Option, + pub stake_hex: Option, + pub stake_bech32: Option, + pub is_script: Option, +} +``` +

diff --git a/book/src/configuration/sources.md b/book/src/configuration/sources.md index 24cbbbe5..b34718c2 100644 --- a/book/src/configuration/sources.md +++ b/book/src/configuration/sources.md @@ -9,8 +9,9 @@ The **Node-to-Client (N2C)** source uses Ouroboros mini-protocols to connect to ## Fields - type: `"N2N" | "N2C"` -- address (*): `""` -- path (**): `""` +- address (*): `String` +- path (**): `String` +- min_depth: `"Option"` (*) Use only with `type = "N2N"` diff --git a/book/src/configuration/storage.md b/book/src/configuration/storage.md index 30c895ac..82a0a5e4 100644 --- a/book/src/configuration/storage.md +++ b/book/src/configuration/storage.md @@ -9,7 +9,8 @@ See our [Redis-cli basics](../guides/redis.md) guide to get started playing with ### Fields - type: `"Redis" | "Skip"` -- connection_params (*): `""` +- connection_params (*): `String` +- cursor_key (*): `Option` (*) Use only with `type = "Redis"` From 87634af155f60bd0bf5017a6d6f29cce7388247d Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Thu, 30 Mar 2023 07:00:42 -0300 Subject: [PATCH 12/14] Improve predicate doc --- book/src/configuration/reducers.md | 94 ++++++++++++++++-------------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/book/src/configuration/reducers.md b/book/src/configuration/reducers.md index 309ec5d3..9da2aa58 100644 --- a/book/src/configuration/reducers.md +++ b/book/src/configuration/reducers.md @@ -21,42 +21,35 @@ ## Predicates -Predicates can be used as filters by many reducers. +Following `Predicate`s are available to be used as filters by some reducers. -``` -pub enum Predicate { - AllOf(Vec), - AnyOf(Vec), - Not(Box), - Block(BlockPattern), - Transaction(TransactionPattern), - InputAddress(AddressPattern), - OutputAddress(AddressPattern), - WithdrawalAddress(AddressPattern), - CollateralAddress(AddressPattern), - Address(AddressPattern), -} -``` +- all_of: `Vec` +- any_of: `Vec` +- not: `Predicate` +- block: `BlockPattern` +- transaction: `TransactionPattern` +- input_address: `AddressPattern` +- output_address: `AddressPattern` +- withdrawal_address: `AddressPattern` +- collateral_address: `AddressPattern` +- address: `AddressPattern` -``` -pub struct BlockPattern { - pub slot_before: Option, - pub slot_after: Option, -} -``` -``` -#[derive(Deserialize, Clone, Default)] -pub struct AddressPattern { - pub exact_hex: Option, - pub exact_bech32: Option, - pub payment_hex: Option, - pub payment_bech32: Option, - pub stake_hex: Option, - pub stake_bech32: Option, - pub is_script: Option, -} -``` +BlockPattern: +- slot_before: `Option` +- slot_after: `Option` + +TransactionPattern: +- is_valid: `Option` + +AddressPattern: +- exact_hex: `Option` +- exact_bech32: `Option` +- payment_hex: `Option` +- payment_bech32: `Option` +- stake_hex: `Option` +- stake_bech32: `Option` +- is_script: `Option`

@@ -86,7 +79,10 @@ pub struct AddressPattern { ### Config - key_prefix: `Option` -- filter: `Option` +- filter (*): `Option` + +(*) See: [Predicates](#predicates) + ### Example ### Output Format @@ -114,14 +110,15 @@ pub struct AddressPattern { ### Config - key_prefix: `Option` -- filter: `Option` -- aggr_by: `Option` +- filter (*): `Option` +- aggr_by (**): `Option` - /// Policies to match - /// - /// If specified only those policy ids as hex will be taken into account, if - /// not all policy ids will be indexed. - policy_ids_hex: `Option>` + +(*) See: [Predicates](#predicates) + +(**) Policies to match. If specified only those policy ids as hex will be taken into account, if not all policy ids will be indexed. + ### Example ### Output Format @@ -133,7 +130,10 @@ pub struct AddressPattern { ### Config - key_prefix: `Option` -- filter: `Option` +- filter (*): `Option` + +(*) See: [Predicates](#predicates) + ### Example ### Output Format @@ -145,7 +145,9 @@ pub struct AddressPattern { ### Config - key_prefix: `Option` -- filter: `Option` +- filter (*): `Option` + +(*) See: [Predicates](#predicates) ### Example ### Output Format @@ -207,9 +209,11 @@ pub struct AddressPattern { ### Config - key_prefix: `Option` -- filter: `Option` +- filter (*): `Option` - projection: `"Cbor" | "Json"` +(*) See: [Predicates](#predicates) + ### Example ``` toml @@ -228,7 +232,9 @@ key_prefix: "c1" ### Config - key_prefix: `Option` -- filter: `Option` +- filter (*): `Option` + +(*) See: [Predicates](#predicates) ### Example ### Output Format From 1c13ca9f6b064e6cc4ea57c6bfb6f4abed5d3c0d Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Thu, 30 Mar 2023 14:52:36 -0300 Subject: [PATCH 13/14] Predicates section --- book/src/SUMMARY.md | 3 +- book/src/configuration/reducers.md | 48 ++++-------------------------- 2 files changed, 8 insertions(+), 43 deletions(-) diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index c8d70aba..f42e523a 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -8,7 +8,8 @@ - [Usage](./usage/README.md) - [Configuration](./configuration/README.md) - [Sources](./configuration/sources.md) - - [Reducers](./configuration/reducers.md) + - [Reducers](./configuration/reducers/README.md) + - [Predicates](./configuration/reducers/predicates.md) - [Storage](./configuration/storage.md) - [Enrich](./configuration/enrich.md) - [Intersect](./configuration/intersect.md) diff --git a/book/src/configuration/reducers.md b/book/src/configuration/reducers.md index 9da2aa58..c0217ef6 100644 --- a/book/src/configuration/reducers.md +++ b/book/src/configuration/reducers.md @@ -19,42 +19,6 @@ - [utxos_by_asset](#utxos_by_asset) -## Predicates - -Following `Predicate`s are available to be used as filters by some reducers. - -- all_of: `Vec` -- any_of: `Vec` -- not: `Predicate` -- block: `BlockPattern` -- transaction: `TransactionPattern` -- input_address: `AddressPattern` -- output_address: `AddressPattern` -- withdrawal_address: `AddressPattern` -- collateral_address: `AddressPattern` -- address: `AddressPattern` - - -BlockPattern: -- slot_before: `Option` -- slot_after: `Option` - -TransactionPattern: -- is_valid: `Option` - -AddressPattern: -- exact_hex: `Option` -- exact_bech32: `Option` -- payment_hex: `Option` -- payment_bech32: `Option` -- stake_hex: `Option` -- stake_bech32: `Option` -- is_script: `Option` - -
-
-
- ## address_by_asset ### Config @@ -81,7 +45,7 @@ AddressPattern: - key_prefix: `Option` - filter (*): `Option` -(*) See: [Predicates](#predicates) +(*) See: [Predicates](./predicates.md) ### Example @@ -115,7 +79,7 @@ AddressPattern: - policy_ids_hex: `Option>` -(*) See: [Predicates](#predicates) +(*) See: [Predicates](./predicates.md) (**) Policies to match. If specified only those policy ids as hex will be taken into account, if not all policy ids will be indexed. @@ -132,7 +96,7 @@ AddressPattern: - key_prefix: `Option` - filter (*): `Option` -(*) See: [Predicates](#predicates) +(*) See: [Predicates](./predicates.md) ### Example ### Output Format @@ -147,7 +111,7 @@ AddressPattern: - key_prefix: `Option` - filter (*): `Option` -(*) See: [Predicates](#predicates) +(*) See: [Predicates](./predicates.md) ### Example ### Output Format @@ -212,7 +176,7 @@ AddressPattern: - filter (*): `Option` - projection: `"Cbor" | "Json"` -(*) See: [Predicates](#predicates) +(*) See: [Predicates](./predicates.md) ### Example @@ -234,7 +198,7 @@ key_prefix: "c1" - key_prefix: `Option` - filter (*): `Option` -(*) See: [Predicates](#predicates) +(*) See: [Predicates](./predicates.md) ### Example ### Output Format From cf9e5782f030a3ffd96a193b54e5aa5adf494d1a Mon Sep 17 00:00:00 2001 From: Franco Cortesi Date: Sun, 2 Apr 2023 14:07:42 -0300 Subject: [PATCH 14/14] Wip doc progress --- book/src/SUMMARY.md | 1 + book/src/configuration/README.md | 8 +- book/src/configuration/enrich.md | 2 +- .../{reducers.md => reducers/README.md} | 0 book/src/configuration/reducers/predicates.md | 401 ++++++++++++++++++ book/src/troubleshooting/README.md | 25 ++ book/src/usage/README.md | 1 + 7 files changed, 435 insertions(+), 3 deletions(-) rename book/src/configuration/{reducers.md => reducers/README.md} (100%) create mode 100644 book/src/configuration/reducers/predicates.md create mode 100644 book/src/troubleshooting/README.md diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index f42e523a..069101c5 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -17,6 +17,7 @@ - [Policy](./configuration/policy.md) - [Advanced Features](./advanced/README.md) - [Swarm Mode](./advanced/swarm_mode.md) +- [Troubleshooting](./troubleshooting/README.md) - [Guides](./guides/README.md) - [Testdrive](./guides/testdrive.md) - [NodeJS Client](./guides/nodejs.md) diff --git a/book/src/configuration/README.md b/book/src/configuration/README.md index 9f5ecb0b..9fded78f 100644 --- a/book/src/configuration/README.md +++ b/book/src/configuration/README.md @@ -1,7 +1,12 @@ # Configuration +For the purpose of testing out Scrolls you can use the provided configuration located in `testdrive/simple/daemon.toml`. See below for another example with explanations and check the following sections of this book to understand in detail each section of the configuration file. -Scrolls daemon must be configured using a single `.toml` file. For the purpose of testing out Scrolls you can use the provided configuration located in `testdrive/simple/daemon.toml`. See below for another example config with explantions and check the following sections of this book to understand in detail each section of the configuration file. +## Format +Scrolls daemon supports `.toml` and `.json` configuration files. Unlike json, toml supports comments which are very handy to turn declarations on and off, specially during early stages of development, debugging, learning, etc. On the other hand, deeply nested filters can be difficult to understand using toml syntax, so the user can choose to declare the whole configuration with json, or instead to rely on tools like [toml2json](https://github.com/woodruffw/toml2json) and [remarshal](https://github.com/remarshal-project/remarshal) to translate small chunks of json (such as complex deeply nested filters) to be used in toml configuration files. +When working with toml configuration files, sometimes it also helps to translate the whole configuration to json, and use [jq](https://stedolan.github.io/jq/)/[bat](https://github.com/sharkdp/bat) to make the json human friendly. This often helps to understand the structure of the filters. Example: `toml2json ./configuration.toml | jq | bat -l json` + +## Configuration Example ```toml # get data from a relay node [source] @@ -41,4 +46,3 @@ value = [57867490, "c491c5006192de2c55a95fb3544f60b96bd1665accaf2dfa2ab12fc7191f type = "Mainnet" ``` -Sometimes it helps to use [toml2json](https://github.com/woodruffw/toml2json) and [jq](https://stedolan.github.io/jq/) if you need to visualize toml files as json. Example: `toml2json ./configuration.toml | jq` diff --git a/book/src/configuration/enrich.md b/book/src/configuration/enrich.md index 5189ba18..346f980c 100644 --- a/book/src/configuration/enrich.md +++ b/book/src/configuration/enrich.md @@ -1,5 +1,5 @@ # Enrich -Store UTXOs information in a local DB, this is needed for some reducers to work. Currently, only [Sled](https://github.com/spacejam/sled) supported. +Store utxo information in a local DB, this is needed for some reducers to work. Currently, only [Sled](https://github.com/spacejam/sled) databases are supported. ## Fields - type: `"Sled" | "Skip"` diff --git a/book/src/configuration/reducers.md b/book/src/configuration/reducers/README.md similarity index 100% rename from book/src/configuration/reducers.md rename to book/src/configuration/reducers/README.md diff --git a/book/src/configuration/reducers/predicates.md b/book/src/configuration/reducers/predicates.md new file mode 100644 index 00000000..0d6e5f42 --- /dev/null +++ b/book/src/configuration/reducers/predicates.md @@ -0,0 +1,401 @@ +# Predicates + +Following `Predicate`s are available to be used as filters by some reducers. + +- [all_of](#all_of-vecpredicate) `Vec` +- [any_of](#any_of-vecpredicate) `Vec` +- [not](#not-predicate) `Predicate` +- [block](#block-blockpattern) `BlockPattern` +- [transaction](#transaction-transactionpattern) `TransactionPattern` +- [input_address](#input_address-addresspattern) `AddressPattern` +- [output_address](#output_address-addresspattern) `AddressPattern` +- [withdrawal_address](#withdrawal_address-addresspattern) `AddressPattern` +- [collateral_address](#collateral_address-addresspattern) `AddressPattern` +- [address](#address-addresspattern) `AddressPattern` + + +##### BlockPattern: +- slot_before: `Option` +- slot_after: `Option` + +##### TransactionPattern: +- is_valid: `Option` + +##### AddressPattern: +- exact_hex: `Option` +- exact_bech32: `Option` +- payment_hex: `Option` +- payment_bech32: `Option` +- stake_hex: `Option` +- stake_bech32: `Option` +- is_script: `Option` + +
+
+
+ +## `all_of (Vec)` + This predicate will yield true when _all_ of the predicates in the argument yields true. + +#### Example (toml) + +```toml +[[reducers]] +type = "AddressByTxo" + +[[reducers.filter.all_of]] +[reducers.filter.all_of.output_address] +exact = "
" + +[[reducers.filter.all_of]] +[reducers.filter.all_of.input_address] +exact = "
" +``` + +#### Example (json) + +```json +{ + "reducers": [ + { + "filter": { + "all_of": [ + { + "output_address": { + "exact": "
" + } + }, + { + "input_address": { + "exact": "
" + } + } + ] + }, + "type": "AddressByTxo" + } + ] +} +``` + +
+
+
+ + +## `any_of (Vec)` + This predicate will yield true when _any_ of the predicates in the argument yields true. + +#### Example (toml) + +```toml +[[reducers]] +type = "AddressByTxo" + +[[reducers.filter.any_of]] +[reducers.filter.any_of.output_address] +exact = "
" + +[[reducers.filter.any_of]] +[reducers.filter.any_of.input_address] +exact = "
" +``` + +#### Example (json) + +```json +{ + "reducers": [ + { + "filter": { + "any_of": [ + { + "output_address": { + "exact": "
" + } + }, + { + "input_address": { + "exact": "
" + } + } + ] + }, + "type": "AddressByTxo" + } + ] +} +``` + +
+
+
+ +## not (Predicate) + This predicate will yield true when the predicate in the arguments yields false. + + +#### Example (toml) + +```toml +[[reducers]] +type = "AddressByTxo" + +[[reducers.filter.not]] +[reducers.filter.not.output_address] +exact = "" + +``` + +#### Example (json) + +```json +{ + "reducers": [ + { + "type": "AddressByTxo", + "filter": { + "not": { + "output_address": { + "exact": "" + } + } + } + } + ] +} +``` + +
+
+
+ +## `block (BlockPattern)` + +#### Example (toml) + +```toml +[[reducers]] +type = "AddresByTxo" + +[[reducers.filter.block]] +slot_before = 83548786 +``` + +#### Example (json) + +```json +{ + "reducers": [ + { + "type": "AddresByTxo", + "filter": { + "block": [ + { + "slot_before": 83548786 + } + ] + } + } + ] +} +``` + +
+
+
+ +## `transaction (TransactionPattern)` + +#### Example (toml) + +```toml +[[reducers]] +type = "AddressByTxo" + +[reducers.filter.transaction] +is_valid = false + +``` + +#### Example (json) + +```json +{ + "reducers": [ + { + "filter": { + "transaction": { + "is_valid": false + } + }, + "type": "AddressByTxo" + } + ] +} +``` + +
+
+
+ +## `input_address (AddressPattern)` + +#### Example (toml) + +```toml +[[reducers]] +type = "AddressByTxo" + +[reducers.filter.input_address] +exact_hex = "" +``` + +#### Example (json) + +```json +{ + "reducers": [ + { + "filter": { + "input_address": { + "exact_hex": "" + } + }, + "type": "AddressByTxo" + } + ] +} +``` + +
+
+
+ +## `output_address (AddressPattern)` + +#### Example (toml) + +```toml +[[reducers]] +type = "AddressByTxo" + +[reducers.filter.output_address] +exact_hex = "" +``` + +#### Example (json) + +```json +{ + "reducers": [ + { + "filter": { + "output_address": { + "exact_hex": "" + } + }, + "type": "AddressByTxo" + } + ] +} +``` + +
+
+
+ +## `withdrawal_address (AddressPattern)` + +#### Example (toml) + +```toml +[[reducers]] +type = "AddressByTxo" + +[reducers.filter.withdrawal_address] +exact_hex = "" +``` + +#### Example (json) + +```json +{ + "reducers": [ + { + "filter": { + "withdrawal_address": { + "exact_hex": "" + } + }, + "type": "AddressByTxo" + } + ] +} +``` + +
+
+
+ +## `collateral_address (AddressPattern)` + +#### Example (toml) + +```toml +[[reducers]] +type = "AddressByTxo" + +[reducers.filter.collateral_address] +exact_hex = "" +``` + +#### Example (json) + +```json +{ + "reducers": [ + { + "filter": { + "collateral_address": { + "exact_hex": "" + } + }, + "type": "AddressByTxo" + } + ] +} +``` + +
+
+
+ +## `address (AddressPattern)` + +#### Example (toml) + +```toml +[[reducers]] +type = "AddressByTxo" + +[reducers.filter.address] +exact_hex = "" +``` + +#### Example (json) + +```json +{ + "reducers": [ + { + "filter": { + "address": { + "exact_hex": "" + } + }, + "type": "AddressByTxo" + } + ] +} +``` diff --git a/book/src/troubleshooting/README.md b/book/src/troubleshooting/README.md new file mode 100644 index 00000000..76827252 --- /dev/null +++ b/book/src/troubleshooting/README.md @@ -0,0 +1,25 @@ +# Troubleshooting + +## Error: Missing utxo + +``` +STAGE: reducers, WORK, PANIC: missing utxo: # +``` + +When processing a particular transaction, some reducers could need inputs to be cached in a local database, this is called [enrichment](../configuration/enrich.md). To avoid facing this error, you may need to run Scrolls from origin or some old [intersection point](../configuration/intersect.md), to be sure you have every needed tx input cached in the local db. + +Alternatively, you may want to skip this error with: + +``` + [policy] +missing_data = "Skip" +``` + +## Error: chain-sync intersect not found + +``` +STAGE: n2n, BOOTSTRAP, PANIC: chain-sync intersect not found +``` + + +Stored `_cursor` value doesn't point to a valid pair of `"absolute_slot,block_hash"` diff --git a/book/src/usage/README.md b/book/src/usage/README.md index 6e856fea..1b52724a 100644 --- a/book/src/usage/README.md +++ b/book/src/usage/README.md @@ -2,6 +2,7 @@ Once you have Scrolls and Redis installed in your system (see [Installation](../installation/index.md)) you can start the daemon with the following command: + ``` bash /path/to/scrolls daemon --config /path/to/config.toml ```