From 62ac984e23f9280862059be22a272711cf842d30 Mon Sep 17 00:00:00 2001 From: JordiParraCrespo Date: Tue, 10 Feb 2026 17:18:16 +0100 Subject: [PATCH 1/2] chore: improve local development setup --- Makefile | 93 +++++++++++++ README.md | 267 +++++++++++++++++++++++++++++++++--- configs/mainnet-config.yaml | 65 +++++++++ configs/testnet-config.yaml | 65 +++++++++ docker-compose.yml | 3 +- hasura/config.yaml | 2 +- 6 files changed, 470 insertions(+), 25 deletions(-) create mode 100644 configs/mainnet-config.yaml create mode 100644 configs/testnet-config.yaml diff --git a/Makefile b/Makefile index 7c642a9c7..c063e5990 100644 --- a/Makefile +++ b/Makefile @@ -93,3 +93,96 @@ format: clean: rm -f tools-stamp ./build/** .PHONY: clean + +############################################################################### +### Config ### +############################################################################### + +update-config: + @echo "Copying $(CONFIG) to ~/.callisto/config.yaml..." + @mkdir -p ~/.callisto + @cp $(CONFIG) ~/.callisto/config.yaml + @echo "Config updated successfully" +.PHONY: update-config + +init-config: + @echo "Initializing callisto config..." + @./build/callisto init +.PHONY: init-config + +############################################################################### +### Database ### +############################################################################### + +DB_CONTAINER := $(shell docker compose ps -q database 2>/dev/null) +DB_EXEC := docker compose exec -T database psql -U user -d database + +db-schema-drop: + @echo "Dropping all tables..." + @$(DB_EXEC) -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" + @echo "All tables dropped" +.PHONY: db-schema-drop + +db-schema-apply: + @echo "Applying schema files..." + @for f in $(sort $(wildcard database/schema/*.sql)); do \ + echo " Applying $$f..."; \ + $(DB_EXEC) < $$f; \ + done + @echo "Schema applied successfully" +.PHONY: db-schema-apply + +db-schema-reset: db-schema-drop db-schema-apply + @echo "Schema reset complete" +.PHONY: db-schema-reset + +db-shell: + @docker compose exec database psql -U user -d database +.PHONY: db-shell + +db-tables: + @$(DB_EXEC) -c "\dt" +.PHONY: db-tables + +db-size: + @$(DB_EXEC) -c "SELECT schemaname, tablename, pg_size_pretty(pg_total_relation_size(schemaname || '.' || tablename)) AS size FROM pg_tables WHERE schemaname = 'public' ORDER BY pg_total_relation_size(schemaname || '.' || tablename) DESC;" +.PHONY: db-size + +############################################################################### +### Local development ### +############################################################################### + +start: update-config build + @echo "Starting database services..." + @docker compose up -d + @echo "Waiting for database to be ready..." + @sleep 3 + @echo "Starting callisto..." + @./build/callisto start +.PHONY: start + +start-clean: update-config build + @echo "Starting database services..." + @docker compose up -d + @echo "Waiting for database to be ready..." + @sleep 3 + @$(MAKE) db-schema-reset + @echo "Starting callisto..." + @./build/callisto start +.PHONY: start-clean + +start-testnet: CONFIG=configs/testnet-config.yaml +start-testnet: start +.PHONY: start-testnet + +start-testnet-clean: CONFIG=configs/testnet-config.yaml +start-testnet-clean: start-clean +.PHONY: start-testnet-clean + +start-mainnet: CONFIG=configs/mainnet-config.yaml +start-mainnet: start +.PHONY: start-mainnet + +start-mainnet-clean: CONFIG=configs/mainnet-config.yaml +start-mainnet-clean: start-clean +.PHONY: start-mainnet-clean diff --git a/README.md b/README.md index d3422a17f..5772b2029 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,256 @@ -# Callisto -[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/forbole/bdjuno/Tests)](https://github.com/forbole/bdjuno/actions?query=workflow%3ATests) -[![Go Report Card](https://goreportcard.com/badge/github.com/forbole/bdjuno)](https://goreportcard.com/report/github.com/forbole/bdjuno) -![Codecov branch](https://img.shields.io/codecov/c/github/forbole/bdjuno/cosmos/v0.40.x) +# XRPL EVM Callisto - Blockchain Indexer -Callisto (formerly BDJuno) is the [Juno](https://github.com/forbole/juno) implementation -for [Big Dipper](https://github.com/forbole/big-dipper). +Callisto is a blockchain indexer (formerly BDJuno) built on the Juno framework designed to index and parse Cosmos blockchain data. It stores blockchain data in PostgreSQL and provides GraphQL APIs through Hasura. -It extends the custom Juno behavior by adding different handlers and custom operations to make it easier for Big Dipper -showing the data inside the UI. +## Requirements -All the chains' data that are queried from the RPC and gRPC endpoints are stored inside -a [PostgreSQL](https://www.postgresql.org/) database on top of which [GraphQL](https://graphql.org/) APIs can then be -created using [Hasura](https://hasura.io/). +- Go 1.23.4 +- PostgreSQL 13+ +- Docker & Docker Compose +- Hasura 2.6.1+ (included in docker-compose) -## Usage -To know how to setup and run Callisto, please refer to -the [docs website](https://docs.bigdipper.live/cosmos-based/parser/overview/). +## Quick Start -## Testing -If you want to test the code, you can do so by running +### 1. Install Go -```shell -$ make test-unit +Using GVM (Go Version Manager): + +```bash +gvm install go1.23.4 +gvm use go1.23.4 +``` + +### 2. Build + +```bash +make build +``` + +The binary will be created at `./build/callisto`. To install globally: + +```bash +make install +``` + +### 3. Configuration + +Running `callisto init` generates a default configuration file at `~/.callisto/config.yaml`: + +```bash +./build/callisto init +``` + +Pre-configured network configs are provided in `configs/`: + +- `configs/testnet-config.yaml` — XRPL EVM Sidechain testnet +- `configs/mainnet-config.yaml` — XRPL EVM Sidechain mainnet + +Copy the desired config before starting: + +```bash +mkdir -p ~/.callisto +cp configs/testnet-config.yaml ~/.callisto/config.yaml +``` + +### 4. Start Indexing + +```bash +# 1. Start PostgreSQL and Hasura +docker compose up -d + +# 2. Build and start the indexer +make build +./build/callisto start +``` + +### 5. Start with Make Targets + +Alternatively, Make targets are available that automate the above steps (build, copy config, start services, and launch the indexer) in a single command: + +```bash +# Testnet +make start-testnet + +# Mainnet +make start-mainnet +``` + +For a clean start (drops and recreates the database schema): + +```bash +make start-testnet-clean +make start-mainnet-clean +``` + +You can also pass a custom config: + +```bash +make start CONFIG=path/to/config.yaml +``` + +### 6. Database Management + +Commands to interact with the database without restarting Docker containers: + +```bash +# Drop all tables and recreate empty schema +make db-schema-drop + +# Apply all schema SQL files (database/schema/*.sql) in order +make db-schema-apply + +# Full reset: drop + apply (wipes data, keeps containers running) +make db-schema-reset + +# Open an interactive psql session +make db-shell + +# List all tables +make db-tables + +# Show size of each table +make db-size ``` -**Note**: Requires [Docker](https://docker.com). +Use `make db-schema-reset` when you need a fresh database without tearing down the Docker stack. The `start-*-clean` targets use this internally. + +## Services + +| Service | URL | Description | +|---------|-----|-------------| +| Hasura Console & GraphQL API | http://localhost:8080 | GraphQL management console and endpoint | +| Callisto Actions | http://localhost:3000 | HTTP server for Hasura actions | +| PostgreSQL | localhost:5432 | Database connection | + +Default database credentials (from docker-compose.yml): +- User: `user` +- Password: `password` +- Database: `database` + +## CLI Commands -This will: -1. Create a Docker container running a PostgreSQL database. -2. Run all the tests using that database as support. +| Command | Description | +|---------|-------------| +| `callisto init` | Generate default config at `~/.callisto/config.yaml` | +| `callisto start` | Start continuous block indexing | +| `callisto version` | Display version information | +| `callisto migrate` | Migrate database between versions | + +## Configuration + +### Actions Module + +The actions module provides an HTTP server for Hasura actions: + +```yaml +actions: + host: "127.0.0.1" + port: 3000 + node: + rpc: "http://localhost:26657" + grpc: "http://localhost:9090" +``` + +### Hasura Configuration + +Located in `hasura/config.yaml`: + +```yaml +version: 3 +endpoint: http://localhost:8080 +metadata_directory: metadata +migrations_directory: migrations +actions: + kind: synchronous + handler_webhook_baseurl: http://localhost:3000 +``` + +## Database Schema + +The database schema is located in `database/schema/` and consists of: + +| File | Module | Purpose | +|------|--------|---------| +| 00-cosmos.sql | Core | Blocks, transactions, validators, messages | +| 01-auth.sql | x/auth | Vesting accounts and periods | +| 02-bank.sql | x/bank | Total supply tracking | +| 03-staking.sql | x/staking | Validators, delegations, staking pool | +| 04-consensus.sql | Consensus | Pre-commits and validator state | +| 05-mint.sql | x/mint | Inflation and minting parameters (excluded, not used in XRPL EVM Sidechain) | +| 06-distribution.sql | x/distribution | Community pool and distribution | +| 07-pricefeed.sql | x/pricefeed | Token prices and market data | +| 08-gov.sql | x/gov | Proposals, votes, deposits | +| 09-modules.sql | Modules | Module tracking | +| 10-slashing.sql | x/slashing | Slashing parameters and signing info | +| 11-feegrant.sql | x/feegrant | Fee grant allowances | +| 12-upgrade.sql | x/upgrade | Software upgrade plans | + +## Development + +### Run Tests + +```bash +# Start test database +make start-docker-test + +# Run unit tests +make test-unit + +# Stop test database +make stop-docker-test +``` + +### Linting and Formatting + +```bash +# Run linter +make lint + +# Auto-fix lint issues +make lint-fix + +# Format code +make format +``` + +### Clean Build + +```bash +make clean +``` + +## Troubleshooting + +### Block height not available + +``` +ERR re-enqueueing failed block err="failed to get block from node: error in json rpc client, with http response metadata: (Status: 200 OK, Protocol HTTP/1.1). RPC error -32603 - Internal error: height 4 is not available, lowest height is 4133001" height=5 +``` + +This error means the node you are connected to has pruned blocks below its lowest available height. To fix it, either: + +1. **Connect to an archive/full node** that has all block heights available by updating the RPC/gRPC endpoints in your config. +2. **Update `start_height`** in your config (`parsing.start_height`) to the lowest height available on the node (e.g. `4133001` in the example above). + +### Hasura image platform mismatch + +``` +The requested image's platform (linux/arm64/v8) does not match the detected host platform (linux/amd64/v3) and no specific platform was requested +``` + +This happens when the Hasura image in `docker-compose.yml` is set to the ARM variant. Update the image from: + +```yaml +image: hasura/graphql-engine:v2.6.1.cli-migrations-v3.ubuntu.arm64 +``` + +to: + +```yaml +image: hasura/graphql-engine:v2.6.1.cli-migrations-v3 +``` +## References +- [Official Documentation](https://docs.bigdipper.live/cosmos-based/parser/overview/) +- [GitHub Repository](https://github.com/forbole/callisto) diff --git a/configs/mainnet-config.yaml b/configs/mainnet-config.yaml new file mode 100644 index 000000000..55fc44cbe --- /dev/null +++ b/configs/mainnet-config.yaml @@ -0,0 +1,65 @@ +chain: + bech32_prefix: ethm + modules: + - modules + - message_type + - messages + - auth + - bank + - consensus + - feegrant + - gov + - slashing + - staking + - actions + - upgrade +node: + type: remote + config: + rpc: + client_name: juno + address: https://cosmos-rpc.xrplevm.org + max_connections: 20 + grpc: + address: cosmos-grpc.xrplevm.org:443 + insecure: false + api: + address: https://cosmos-api.xrplevm.org +parsing: + workers: 1 + start_height: 1 + average_block_time: 4s + listen_new_blocks: true + parse_old_blocks: true + parse_genesis: true +database: + url: postgresql://user:password@localhost:5432/database?sslmode=disable&search_path=public + max_open_connections: 1 + max_idle_connections: 1 + partition_size: 100000 + partition_batch: 1000 + ssl_mode_enable: "false" + ssl_root_cert: "" + ssl_cert: "" + ssl_key: "" +logging: + level: debug + format: text +pricefeed: + tokens: + - name: PoA + units: + - denom: apoa + exponent: 0 + - denom: poa + exponent: 6 + price_id: cosmos + - name: XRP + units: + - denom: axrp + exponent: 0 + - denom: xrp + exponent: 6 +actions: + host: 127.0.0.1 + port: 3000 diff --git a/configs/testnet-config.yaml b/configs/testnet-config.yaml new file mode 100644 index 000000000..f79a67a7d --- /dev/null +++ b/configs/testnet-config.yaml @@ -0,0 +1,65 @@ +chain: + bech32_prefix: ethm + modules: + - modules + - message_type + - messages + - auth + - bank + - consensus + - feegrant + - gov + - slashing + - staking + - actions + - upgrade +node: + type: remote + config: + rpc: + client_name: juno + address: https://cosmos-rpc.testnet.xrplevm.org + max_connections: 20 + grpc: + address: cosmos-grpc.testnet.xrplevm.org:443 + insecure: false + api: + address: https://cosmos-api.testnet.xrplevm.org +parsing: + workers: 1 + start_height: 5405153 + average_block_time: 4s + listen_new_blocks: true + parse_old_blocks: true + parse_genesis: true +database: + url: postgresql://user:password@localhost:5432/database?sslmode=disable&search_path=public + max_open_connections: 1 + max_idle_connections: 1 + partition_size: 100000 + partition_batch: 1000 + ssl_mode_enable: "false" + ssl_root_cert: "" + ssl_cert: "" + ssl_key: "" +logging: + level: debug + format: text +pricefeed: + tokens: + - name: PoA + units: + - denom: apoa + exponent: 0 + - denom: poa + exponent: 6 + price_id: cosmos + - name: XRP + units: + - denom: axrp + exponent: 0 + - denom: xrp + exponent: 6 +actions: + host: 127.0.0.1 + port: 3000 diff --git a/docker-compose.yml b/docker-compose.yml index 027de0f67..b607e471b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,11 +11,10 @@ services: ## enable debugging mode. It is recommended to disable this in production HASURA_GRAPHQL_DEV_MODE: "true" HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log - ACTION_BASE_URL: http://localhost:3000 + ACTION_BASE_URL: http://host.docker.internal:3000 volumes: - ./hasura/metadata:/hasura-metadata ports: - - "3000:3000" - "8080:8080" database: diff --git a/hasura/config.yaml b/hasura/config.yaml index 44771cf30..740ec7468 100644 --- a/hasura/config.yaml +++ b/hasura/config.yaml @@ -13,7 +13,7 @@ migrations_directory: migrations seeds_directory: seeds actions: kind: synchronous - handler_webhook_baseurl: http://localhost:3000 + handler_webhook_baseurl: http://host.docker.internal:3000 codegen: framework: "" output_dir: "" From ffee6aad434c28a855cd5ef22401e05793d6f13f Mon Sep 17 00:00:00 2001 From: JordiParraCrespo Date: Tue, 10 Feb 2026 17:35:52 +0100 Subject: [PATCH 2/2] fix: autodetect platform and update hasura docker compose based on that --- .gitignore | 1 + Makefile | 15 +++++++++++++-- docker-compose.yml | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 8bad51a91..545427612 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ vendor/ *.toml *.json config.yaml +.env # Coverage coverage.* diff --git a/Makefile b/Makefile index c063e5990..42ce3b650 100644 --- a/Makefile +++ b/Makefile @@ -152,7 +152,18 @@ db-size: ### Local development ### ############################################################################### -start: update-config build +setup-env: + @ARCH=$$(uname -m); \ + if [ "$$ARCH" = "arm64" ] || [ "$$ARCH" = "aarch64" ]; then \ + echo "HASURA_IMAGE_SUFFIX=.ubuntu.arm64" > .env; \ + echo "Detected ARM architecture"; \ + else \ + echo "HASURA_IMAGE_SUFFIX=" > .env; \ + echo "Detected x86 architecture"; \ + fi +.PHONY: setup-env + +start: setup-env update-config build @echo "Starting database services..." @docker compose up -d @echo "Waiting for database to be ready..." @@ -161,7 +172,7 @@ start: update-config build @./build/callisto start .PHONY: start -start-clean: update-config build +start-clean: setup-env update-config build @echo "Starting database services..." @docker compose up -d @echo "Waiting for database to be ready..." diff --git a/docker-compose.yml b/docker-compose.yml index b607e471b..cde6bb709 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.6" services: hasura: - image: hasura/graphql-engine:v2.6.1.cli-migrations-v3.ubuntu.arm64 + image: hasura/graphql-engine:v2.6.1.cli-migrations-v3${HASURA_IMAGE_SUFFIX:-} restart: always environment: ## this env var can be used to add the above postgres database to Hasura as a data source. this can be removed/updated based on your needs