From c8152f5fcb236b06ca98273dcd18fd00fd820f73 Mon Sep 17 00:00:00 2001 From: banasa44 Date: Fri, 11 Apr 2025 16:16:35 +0000 Subject: [PATCH 01/17] feat(indexer): WiP update address_coin/token_balances --- .devcontainer/.blockscout_config.example | 61 ------------------- .../transform/address_coin_balances.ex | 18 ++++-- .../transform/address_token_balances.ex | 26 ++++++++ docker-compose/envs/common-blockscout.env | 10 +-- 4 files changed, 45 insertions(+), 70 deletions(-) delete mode 100644 .devcontainer/.blockscout_config.example diff --git a/.devcontainer/.blockscout_config.example b/.devcontainer/.blockscout_config.example deleted file mode 100644 index 737933b067b6..000000000000 --- a/.devcontainer/.blockscout_config.example +++ /dev/null @@ -1,61 +0,0 @@ -CHAIN_TYPE=ethereum - -ETHEREUM_JSONRPC_VARIANT=geth -ETHEREUM_JSONRPC_TRACE_URL="" - -API_RATE_LIMIT=100 -HEART_BEAT_TIMEOUT=30 -TXS_STATS_DAYS_TO_COMPILE_AT_INIT=2 -INDEXER_MEMORY_LIMIT=6 - -POOL_SIZE=50 -POOL_SIZE_API=50 -ACCOUNT_POOL_SIZE=10 - -INDEXER_DISABLE_EMPTY_BLOCKS_SANITIZER='true' -INDEXER_DISABLE_PENDING_TRANSACTIONS_FETCHER='true' -INDEXER_DISABLE_INTERNAL_TRANSACTIONS_FETCHER='true' -INDEXER_DISABLE_BLOCK_REWARD_FETCHER='true' -INDEXER_DISABLE_ADDRESS_COIN_BALANCE_FETCHER='true' -INDEXER_DISABLE_CATALOGED_TOKEN_UPDATER_FETCHER='true' -ETHEREUM_JSONRPC_DISABLE_ARCHIVE_BALANCES='true' -INDEXER_DISABLE_TOKEN_INSTANCE_RETRY_FETCHER='true' -INDEXER_DISABLE_TOKEN_INSTANCE_REALTIME_FETCHER='true' -INDEXER_DISABLE_TOKEN_INSTANCE_SANITIZE_FETCHER='true' -INDEXER_DISABLE_WITHDRAWALS_FETCHER='true' - -INDEXER_CATCHUP_BLOCKS_BATCH_SIZE=5 -INDEXER_COIN_BALANCES_BATCH_SIZE=1 -INDEXER_EMPTY_BLOCKS_SANITIZER_BATCH_SIZE=1 -INDEXER_BLOCK_REWARD_BATCH_SIZE=1 -INDEXER_RECEIPTS_BATCH_SIZE=10 -INDEXER_COIN_BALANCES_BATCH_SIZE=1 -INDEXER_TOKEN_BALANCES_BATCH_SIZE=1 - -INDEXER_CATCHUP_BLOCKS_CONCURRENCY=1 -MIGRATION_TOKEN_INSTANCE_OWNER_BATCH_SIZE=1 -MIGRATION_TOKEN_INSTANCE_OWNER_CONCURRENCY=1 -INDEXER_BLOCK_REWARD_CONCURRENCY=1 -INDEXER_RECEIPTS_CONCURRENCY=1 -INDEXER_COIN_BALANCES_CONCURRENCY=1 -INDEXER_TOKEN_CONCURRENCY=1 -INDEXER_TOKEN_BALANCES_CONCURRENCY=1 -INDEXER_TOKEN_INSTANCE_RETRY_CONCURRENCY=1 -INDEXER_TOKEN_INSTANCE_REALTIME_CONCURRENCY=1 -INDEXER_TOKEN_INSTANCE_SANITIZE_CONCURRENCY=1 -INDEXER_TOKEN_INSTANCE_RETRY_BATCH_SIZE=1 -INDEXER_TOKEN_INSTANCE_REALTIME_BATCH_SIZE=1 -INDEXER_TOKEN_INSTANCE_SANITIZE_BATCH_SIZE=1 - -INDEXER_TOKEN_BALANCES_FETCHER_INIT_QUERY_LIMIT=2 -INDEXER_COIN_BALANCES_FETCHER_INIT_QUERY_LIMIT=2 - -DISABLE_MARKET='true' -SOURCIFY_INTEGRATION_ENABLED='false' - -API_V2_ENABLED=true - -DISABLE_CATCHUP_INDEXER='false' -INDEXER_CATCHUP_BLOCKS_BATCH_SIZE=10 -INDEXER_CATCHUP_BLOCKS_CONCURRENCY=10 -ETHEREUM_JSONRPC_HTTP_URL="https://ethereum-sepolia-rpc.publicnode.com" diff --git a/apps/indexer/lib/indexer/transform/address_coin_balances.ex b/apps/indexer/lib/indexer/transform/address_coin_balances.ex index 3787261d552f..39bcd6d73017 100644 --- a/apps/indexer/lib/indexer/transform/address_coin_balances.ex +++ b/apps/indexer/lib/indexer/transform/address_coin_balances.ex @@ -6,6 +6,9 @@ defmodule Indexer.Transform.AddressCoinBalances do alias Explorer.Chain.TokenTransfer + @native_token_address System.get_env("NATIVE_TOKEN_ADDRESS") + + def params_set(%{} = import_options) do Enum.reduce(import_options, MapSet.new(), &reducer/2) end @@ -29,11 +32,16 @@ defmodule Indexer.Transform.AddressCoinBalances do defp reducer({:logs_params, logs_params}, acc) when is_list(logs_params) do # a log MUST have address_hash and block_number logs_params - |> Enum.reject( - &(&1.first_topic == TokenTransfer.constant() or - &1.first_topic == TokenTransfer.erc1155_single_transfer_signature() or - &1.first_topic == TokenTransfer.erc1155_batch_transfer_signature()) - ) + |> Enum.reject( + &( + ( + (&1.first_topic == TokenTransfer.constant() or + &1.first_topic == TokenTransfer.erc1155_single_transfer_signature() or + &1.first_topic == TokenTransfer.erc1155_batch_transfer_signature()) + and &1.address_hash != @native_token_address + ) + ) + ) |> Enum.into(acc, fn %{address_hash: address_hash, block_number: block_number} when is_binary(address_hash) and is_integer(block_number) -> diff --git a/apps/indexer/lib/indexer/transform/address_token_balances.ex b/apps/indexer/lib/indexer/transform/address_token_balances.ex index 288757d313c4..5e4478f060f2 100644 --- a/apps/indexer/lib/indexer/transform/address_token_balances.ex +++ b/apps/indexer/lib/indexer/transform/address_token_balances.ex @@ -33,6 +33,32 @@ defmodule Indexer.Transform.AddressTokenBalances do end) end + defp reducer({:transactions_params, transactions_params}, initial) when is_list(transactions_params) do + # Here, we assume that a transaction with a non-zero "value" is a native transfer. + transactions_params + |> Enum.filter(fn %{value: value} -> + # You may need to adjust this check according to how "value" is represented (e.g. Decimal.zero?/1) + value && not Decimal.equal?(value, Decimal.new(0)) + end) + |> Enum.reduce(initial, fn %{ + block_number: block_number, + from_address_hash: from_address, + to_address_hash: to_address, + value: amount + }, + acc -> + # For your native token, assume its token_contract_address is your known native ERC-20 token. + token_contract_address = System.get_env("NATIVE_TOKEN_ADDRESS") + token_type = "ERC-20" + + # Update balances for both sender and receiver. + acc + |> add_token_balance_address(from_address, token_contract_address, nil, token_type, block_number) + |> add_token_balance_address(to_address, token_contract_address, nil, token_type, block_number) + # You might want to store the amount as well in your database or merge it with existing amounts. + end) + end + defp add_token_balance_address(map_set, unquote(burn_address_hash_string()), _, _, _, _), do: map_set defp add_token_balance_address(map_set, address, token_contract_address, token_id, token_type, block_number) do diff --git a/docker-compose/envs/common-blockscout.env b/docker-compose/envs/common-blockscout.env index 491adf4a7ad4..b0656a2ebec2 100644 --- a/docker-compose/envs/common-blockscout.env +++ b/docker-compose/envs/common-blockscout.env @@ -38,7 +38,7 @@ ETHEREUM_JSONRPC_TRACE_URL=http://host.docker.internal:8545/ SECRET_KEY_BASE=56NtB48ear7+wMSf0IQuWDAAazhpb31qyc7GiyspBP2vh7t5zlCsF5QDv76chXeN # CHECK_ORIGIN= PORT=4000 -COIN_NAME= +COIN_NAME=XRP # METADATA_CONTRACT= # VALIDATORS_CONTRACT= # KEYS_MANAGER_CONTRACT= @@ -47,7 +47,9 @@ COIN_NAME= # CHAIN_SPEC_PATH= # CHAIN_SPEC_PROCESSING_DELAY= # SUPPLY_MODULE= -COIN= +COIN=XRP +NATIVE_TOKEN_ADDRESS=0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE +NATIVE_TOKEN_ID=0xba5a21ca88ef6bba2bfff5088994f90e1077e2a1cc3dcc38bd261f00fce2824f DISABLE_MARKET=true # MARKET_NATIVE_COIN_SOURCE= # MARKET_SECONDARY_COIN_SOURCE= @@ -343,8 +345,8 @@ API_V1_WRITE_METHODS_DISABLED=false # INDEXER_ROOTSTOCK_DATA_FETCHER_BATCH_SIZE= # INDEXER_ROOTSTOCK_DATA_FETCHER_CONCURRENCY= # INDEXER_ROOTSTOCK_DATA_FETCHER_DB_BATCH_SIZE= -# INDEXER_BEACON_RPC_URL=http://localhost:5052 -# INDEXER_DISABLE_BEACON_BLOB_FETCHER= +INDEXER_BEACON_RPC_URL=https://rpc.testnet.xrplevm.org +INDEXER_DISABLE_BEACON_BLOB_FETCHER=true # INDEXER_BEACON_BLOB_FETCHER_SLOT_DURATION=12 # INDEXER_BEACON_BLOB_FETCHER_REFERENCE_SLOT=8000000 # INDEXER_BEACON_BLOB_FETCHER_REFERENCE_TIMESTAMP=1702824023 From 570b83c4ece40526409c872a07bb2e658e7f309e Mon Sep 17 00:00:00 2001 From: banasa44 Date: Mon, 14 Apr 2025 13:17:35 +0000 Subject: [PATCH 02/17] feat(indexer): coin balance successfully updated when token transfer --- apps/block_scout_web/assets/package-lock.json | 5 +++-- apps/indexer/lib/indexer/transform/address_token_balances.ex | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/block_scout_web/assets/package-lock.json b/apps/block_scout_web/assets/package-lock.json index c484b41945c8..a227fb5a0d57 100644 --- a/apps/block_scout_web/assets/package-lock.json +++ b/apps/block_scout_web/assets/package-lock.json @@ -102,10 +102,11 @@ } }, "../../../deps/phoenix": { - "version": "0.0.1" + "version": "1.5.14", + "license": "MIT" }, "../../../deps/phoenix_html": { - "version": "0.0.1" + "version": "3.3.4" }, "node_modules/@aashutoshrathi/word-wrap": { "version": "1.2.6", diff --git a/apps/indexer/lib/indexer/transform/address_token_balances.ex b/apps/indexer/lib/indexer/transform/address_token_balances.ex index 5e4478f060f2..d6e812b29f9e 100644 --- a/apps/indexer/lib/indexer/transform/address_token_balances.ex +++ b/apps/indexer/lib/indexer/transform/address_token_balances.ex @@ -44,7 +44,6 @@ defmodule Indexer.Transform.AddressTokenBalances do block_number: block_number, from_address_hash: from_address, to_address_hash: to_address, - value: amount }, acc -> # For your native token, assume its token_contract_address is your known native ERC-20 token. From 80dc227050b13b3dd55f5fccb95d1d33d2a3ea6c Mon Sep 17 00:00:00 2001 From: banasa44 Date: Wed, 16 Apr 2025 15:12:17 +0000 Subject: [PATCH 03/17] feat(indexer): add Indexer.Transformers.TokenToCoinBalanceTransformer and implemented in import_token_balances --- .../lib/indexer/fetcher/token_balance.ex | 58 ++++++++++++------- .../xrpl-evm/token_to_coin_balance.ex | 26 +++++++++ 2 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex diff --git a/apps/indexer/lib/indexer/fetcher/token_balance.ex b/apps/indexer/lib/indexer/fetcher/token_balance.ex index 8b2692707892..0f770d6ed393 100644 --- a/apps/indexer/lib/indexer/fetcher/token_balance.ex +++ b/apps/indexer/lib/indexer/fetcher/token_balance.ex @@ -206,31 +206,47 @@ defmodule Indexer.Fetcher.TokenBalance do Timex.shift(Timex.now(), seconds: value) end - def import_token_balances(token_balances_params) do - addresses_params = format_and_filter_address_params(token_balances_params) - formatted_token_balances_params = format_and_filter_token_balance_params(token_balances_params) - - import_params = %{ - addresses: %{params: addresses_params}, - address_token_balances: %{params: formatted_token_balances_params}, - address_current_token_balances: %{ - params: TokenBalances.to_address_current_token_balances(formatted_token_balances_params) - }, - timeout: @timeout - } +def import_token_balances(token_balances_params) do + # Existing formatting and filtering steps + addresses_params = format_and_filter_address_params(token_balances_params) + formatted_token_balances_params = format_and_filter_token_balance_params(token_balances_params) + + # Apply transformer on the formatted token balances to get potential coin balance entries. + transformed_params = + Indexer.Transformers.TokenToCoinBalanceTransformer.transform_address_token_balances(formatted_token_balances_params) + + # Separate into token balance entries and coin balance entries. + {final_token_balances, coin_balance_entries} = + Enum.split_with(transformed_params, fn + {:address_coin_balance, _} -> false + _ -> true + end) - case Chain.import(import_params) do - {:ok, _} -> - :ok + # Process current token balances as before, using output from the transformer. + final_current_token_balances = TokenBalances.to_address_current_token_balances(final_token_balances) + + # Build import params, now including coin balances. + import_params = %{ + addresses: %{params: addresses_params}, + address_token_balances: %{params: final_token_balances}, + address_current_token_balances: %{params: final_current_token_balances}, + address_coin_balances: + %{params: Enum.map(coin_balance_entries, fn {:address_coin_balance, coin_balance} -> coin_balance end)}, + timeout: @timeout + } + + case Chain.import(import_params) do + {:ok, _} -> + :ok - {:error, reason} -> - Logger.debug(fn -> ["failed to import token balances: ", inspect(reason)] end, - error_count: Enum.count(token_balances_params) - ) + {:error, reason} -> + Logger.debug(fn -> ["failed to import token balances: ", inspect(reason)] end, + error_count: Enum.count(token_balances_params) + ) - :error - end + :error end +end defp format_and_filter_address_params(token_balances_params) do token_balances_params diff --git a/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex new file mode 100644 index 000000000000..bb1e8e258883 --- /dev/null +++ b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex @@ -0,0 +1,26 @@ +defmodule Indexer.Transformers.TokenToCoinBalanceTransformer do + alias Explorer.Chain.AddressCoinBalance + alias Indexer.Constants + + @doc """ + Transforms changes to `address_token_balances` into changes for `address_coin_balances` + if the token contract address hash matches the hardcoded value. + """ + def transform_address_token_balances(params) do + native_token_address = System.get_env("NATIVE_TOKEN_ADDRESS") + + Enum.flat_map(params, fn token_balance -> + if token_balance[:token_contract_address_hash] == native_token_address do + # Create a corresponding address_coin_balance entry + coin_balance = %{ + address_hash: token_balance[:address_hash], + value: token_balance[:value] + } + + [token_balance, {:address_coin_balance, coin_balance}] + else + [token_balance] + end + end) + end +end From e8c97b17343918490dca4f7d4790e176d1f115ef Mon Sep 17 00:00:00 2001 From: banasa44 Date: Mon, 21 Apr 2025 08:30:17 +0000 Subject: [PATCH 04/17] feat(indexer): clean unused/wrong code --- .devcontainer/bin/bs | 2 +- .../transform/address_coin_balances.ex | 7 ++---- .../transform/address_token_balances.ex | 25 ------------------- .../xrpl-evm/token_to_coin_balance.ex | 2 -- 4 files changed, 3 insertions(+), 33 deletions(-) diff --git a/.devcontainer/bin/bs b/.devcontainer/bin/bs index 145c152531dc..90900b07d879 100755 --- a/.devcontainer/bin/bs +++ b/.devcontainer/bin/bs @@ -203,7 +203,7 @@ initialize_db() { # Define the compile subroutine compile() { - mix compile + MIX_DEBUG=1 mix compile } # Define the recompile subroutine diff --git a/apps/indexer/lib/indexer/transform/address_coin_balances.ex b/apps/indexer/lib/indexer/transform/address_coin_balances.ex index 39bcd6d73017..09c885e025e2 100644 --- a/apps/indexer/lib/indexer/transform/address_coin_balances.ex +++ b/apps/indexer/lib/indexer/transform/address_coin_balances.ex @@ -6,8 +6,6 @@ defmodule Indexer.Transform.AddressCoinBalances do alias Explorer.Chain.TokenTransfer - @native_token_address System.get_env("NATIVE_TOKEN_ADDRESS") - def params_set(%{} = import_options) do Enum.reduce(import_options, MapSet.new(), &reducer/2) @@ -35,10 +33,9 @@ defmodule Indexer.Transform.AddressCoinBalances do |> Enum.reject( &( ( - (&1.first_topic == TokenTransfer.constant() or + &1.first_topic == TokenTransfer.constant() or &1.first_topic == TokenTransfer.erc1155_single_transfer_signature() or - &1.first_topic == TokenTransfer.erc1155_batch_transfer_signature()) - and &1.address_hash != @native_token_address + &1.first_topic == TokenTransfer.erc1155_batch_transfer_signature() ) ) ) diff --git a/apps/indexer/lib/indexer/transform/address_token_balances.ex b/apps/indexer/lib/indexer/transform/address_token_balances.ex index d6e812b29f9e..288757d313c4 100644 --- a/apps/indexer/lib/indexer/transform/address_token_balances.ex +++ b/apps/indexer/lib/indexer/transform/address_token_balances.ex @@ -33,31 +33,6 @@ defmodule Indexer.Transform.AddressTokenBalances do end) end - defp reducer({:transactions_params, transactions_params}, initial) when is_list(transactions_params) do - # Here, we assume that a transaction with a non-zero "value" is a native transfer. - transactions_params - |> Enum.filter(fn %{value: value} -> - # You may need to adjust this check according to how "value" is represented (e.g. Decimal.zero?/1) - value && not Decimal.equal?(value, Decimal.new(0)) - end) - |> Enum.reduce(initial, fn %{ - block_number: block_number, - from_address_hash: from_address, - to_address_hash: to_address, - }, - acc -> - # For your native token, assume its token_contract_address is your known native ERC-20 token. - token_contract_address = System.get_env("NATIVE_TOKEN_ADDRESS") - token_type = "ERC-20" - - # Update balances for both sender and receiver. - acc - |> add_token_balance_address(from_address, token_contract_address, nil, token_type, block_number) - |> add_token_balance_address(to_address, token_contract_address, nil, token_type, block_number) - # You might want to store the amount as well in your database or merge it with existing amounts. - end) - end - defp add_token_balance_address(map_set, unquote(burn_address_hash_string()), _, _, _, _), do: map_set defp add_token_balance_address(map_set, address, token_contract_address, token_id, token_type, block_number) do diff --git a/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex index bb1e8e258883..185b24990c41 100644 --- a/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex +++ b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex @@ -1,6 +1,4 @@ defmodule Indexer.Transformers.TokenToCoinBalanceTransformer do - alias Explorer.Chain.AddressCoinBalance - alias Indexer.Constants @doc """ Transforms changes to `address_token_balances` into changes for `address_coin_balances` From efd163881c9731c86fc691e89f3d3d5836be4409 Mon Sep 17 00:00:00 2001 From: banasa44 Date: Mon, 21 Apr 2025 10:09:44 +0000 Subject: [PATCH 05/17] feat(indexer): coin balance updates, update native_token balancees --- .../indexer/fetcher/coin_balance/helper.ex | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex index 80e1689369c5..d727cb715e6d 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex @@ -12,6 +12,7 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do alias Explorer.Chain.Cache.{Accounts, BlockNumber} alias Explorer.Chain.Hash alias Indexer.BufferedTask + alias Indexer.TokenBalances @doc false # credo:disable-for-next-line Credo.Check.Design.DuplicatedCode @@ -93,25 +94,52 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do end) end + + def import_fetched_balances(params_list, broadcast_type \\ false) do value_fetched_at = DateTime.utc_now() importable_balances_params = Enum.map(params_list, &Map.put(&1, :value_fetched_at, value_fetched_at)) - json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments) - importable_balances_daily_params = balances_daily_params(params_list, json_rpc_named_arguments) - addresses_params = balances_params_to_address_params(importable_balances_params) + native_token_address = System.get_env("NATIVE_TOKEN_ADDRESS") + + token_type = + case native_token_address do + nil -> "erc20" + _ -> + {:ok, address_struct} = Hash.Address.cast(native_token_address) + Chain.get_token_type(address_struct) || "erc20" + end + + token_balance_params = + Enum.map(importable_balances_params, fn %{address_hash: address_hash, value: value, block_number: block_number} -> + %{ + token_contract_address_hash: native_token_address, + address_hash: address_hash, + block_number: block_number, + value: value, + token_type: token_type, + token_id: 0, + value_fetched_at: value_fetched_at + } + end) + + current_token_balance_params = TokenBalances.to_address_current_token_balances(token_balance_params) Chain.import(%{ addresses: %{params: addresses_params, with: :balance_changeset}, address_coin_balances: %{params: importable_balances_params}, address_coin_balances_daily: %{params: importable_balances_daily_params}, + address_token_balances: %{params: token_balance_params}, + address_current_token_balances: %{params: current_token_balance_params}, broadcast: broadcast_type }) end + + def import_fetched_daily_balances(params_list, broadcast_type \\ false) do value_fetched_at = DateTime.utc_now() From 75c8b9999a7c79d03f4d7b9a0621f1e1d4c415c8 Mon Sep 17 00:00:00 2001 From: banasa44 Date: Mon, 21 Apr 2025 11:24:28 +0000 Subject: [PATCH 06/17] refactor(indexer): coin balance updates, update native_token balancees --- .../indexer/fetcher/coin_balance/helper.ex | 32 +++++++------------ .../xrpl-evm/coin_to_token_balance.ex | 27 ++++++++++++++++ 2 files changed, 39 insertions(+), 20 deletions(-) create mode 100644 apps/indexer/lib/indexer/transform/xrpl-evm/coin_to_token_balance.ex diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex index d727cb715e6d..ead693b4dd7c 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex @@ -13,6 +13,8 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do alias Explorer.Chain.Hash alias Indexer.BufferedTask alias Indexer.TokenBalances + alias Indexer.Transformers.CoinToTokenBalanceTransformer + @doc false # credo:disable-for-next-line Credo.Check.Design.DuplicatedCode @@ -103,34 +105,24 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments) importable_balances_daily_params = balances_daily_params(params_list, json_rpc_named_arguments) addresses_params = balances_params_to_address_params(importable_balances_params) - native_token_address = System.get_env("NATIVE_TOKEN_ADDRESS") - token_type = - case native_token_address do - nil -> "erc20" - _ -> - {:ok, address_struct} = Hash.Address.cast(native_token_address) - Chain.get_token_type(address_struct) || "erc20" - end + # Transform coin balances to also produce token balances for the native token + transformed_params = CoinToTokenBalanceTransformer.transform_address_coin_balances(importable_balances_params) - token_balance_params = - Enum.map(importable_balances_params, fn %{address_hash: address_hash, value: value, block_number: block_number} -> - %{ - token_contract_address_hash: native_token_address, - address_hash: address_hash, - block_number: block_number, - value: value, - token_type: token_type, - token_id: 0, - value_fetched_at: value_fetched_at - } + {final_coin_balances, token_balance_entries} = + Enum.split_with(transformed_params, fn + {:address_token_balance, _} -> false + _ -> true end) + token_balance_params = + Enum.map(token_balance_entries, fn {:address_token_balance, token_balance} -> token_balance end) + current_token_balance_params = TokenBalances.to_address_current_token_balances(token_balance_params) Chain.import(%{ addresses: %{params: addresses_params, with: :balance_changeset}, - address_coin_balances: %{params: importable_balances_params}, + address_coin_balances: %{params: final_coin_balances}, address_coin_balances_daily: %{params: importable_balances_daily_params}, address_token_balances: %{params: token_balance_params}, address_current_token_balances: %{params: current_token_balance_params}, diff --git a/apps/indexer/lib/indexer/transform/xrpl-evm/coin_to_token_balance.ex b/apps/indexer/lib/indexer/transform/xrpl-evm/coin_to_token_balance.ex new file mode 100644 index 000000000000..cb6b99e26f77 --- /dev/null +++ b/apps/indexer/lib/indexer/transform/xrpl-evm/coin_to_token_balance.ex @@ -0,0 +1,27 @@ +defmodule Indexer.Transformers.CoinToTokenBalanceTransformer do + @doc """ + Transforms changes to `address_coin_balances` into changes for `address_token_balances` + if the native token address is set. + """ + def transform_address_coin_balances(params) do + native_token_address = System.get_env("NATIVE_TOKEN_ADDRESS") + + Enum.flat_map(params, fn coin_balance -> + if native_token_address do + token_balance = %{ + token_contract_address_hash: native_token_address, + address_hash: coin_balance[:address_hash], + block_number: coin_balance[:block_number], + value: coin_balance[:value], + token_type: "erc20", + token_id: 0, + value_fetched_at: coin_balance[:value_fetched_at] + } + + [coin_balance, {:address_token_balance, token_balance}] + else + [coin_balance] + end + end) + end +end From f04437373182dd26962181431bb1fb8fb18b4b61 Mon Sep 17 00:00:00 2001 From: banasa44 Date: Mon, 21 Apr 2025 11:28:42 +0000 Subject: [PATCH 07/17] chore(indexer): clean code, remove unnecessary parenthesis --- .../indexer/transform/address_coin_balances.ex | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/apps/indexer/lib/indexer/transform/address_coin_balances.ex b/apps/indexer/lib/indexer/transform/address_coin_balances.ex index 09c885e025e2..5bab3ccb80d8 100644 --- a/apps/indexer/lib/indexer/transform/address_coin_balances.ex +++ b/apps/indexer/lib/indexer/transform/address_coin_balances.ex @@ -30,15 +30,12 @@ defmodule Indexer.Transform.AddressCoinBalances do defp reducer({:logs_params, logs_params}, acc) when is_list(logs_params) do # a log MUST have address_hash and block_number logs_params - |> Enum.reject( - &( - ( - &1.first_topic == TokenTransfer.constant() or - &1.first_topic == TokenTransfer.erc1155_single_transfer_signature() or - &1.first_topic == TokenTransfer.erc1155_batch_transfer_signature() - ) - ) - ) + + |> Enum.reject( + &(&1.first_topic == TokenTransfer.constant() or + &1.first_topic == TokenTransfer.erc1155_single_transfer_signature() or + &1.first_topic == TokenTransfer.erc1155_batch_transfer_signature()) + ) |> Enum.into(acc, fn %{address_hash: address_hash, block_number: block_number} when is_binary(address_hash) and is_integer(block_number) -> From ce8a5fb590849494ec619ebd26d9a328ab42f43b Mon Sep 17 00:00:00 2001 From: banasa44 Date: Mon, 21 Apr 2025 13:51:35 +0000 Subject: [PATCH 08/17] feat(indexer): add test for updating balances when a token transfer happens. Modify functions according to tests --- .../xrpl-evm/coin_to_token_balance.ex | 2 +- .../xrpl-evm/token_to_coin_balance.ex | 4 +- .../indexer/fetcher/token_balance_test.exs | 44 +++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/apps/indexer/lib/indexer/transform/xrpl-evm/coin_to_token_balance.ex b/apps/indexer/lib/indexer/transform/xrpl-evm/coin_to_token_balance.ex index cb6b99e26f77..8894ae19d34f 100644 --- a/apps/indexer/lib/indexer/transform/xrpl-evm/coin_to_token_balance.ex +++ b/apps/indexer/lib/indexer/transform/xrpl-evm/coin_to_token_balance.ex @@ -13,7 +13,7 @@ defmodule Indexer.Transformers.CoinToTokenBalanceTransformer do address_hash: coin_balance[:address_hash], block_number: coin_balance[:block_number], value: coin_balance[:value], - token_type: "erc20", + token_type: "ERC-20", token_id: 0, value_fetched_at: coin_balance[:value_fetched_at] } diff --git a/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex index 185b24990c41..bdad9efdf5ce 100644 --- a/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex +++ b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex @@ -12,7 +12,9 @@ defmodule Indexer.Transformers.TokenToCoinBalanceTransformer do # Create a corresponding address_coin_balance entry coin_balance = %{ address_hash: token_balance[:address_hash], - value: token_balance[:value] + value: token_balance[:value], + block_number: token_balance[:block_number], + value_fetched_at: token_balance[:value_fetched_at] } [token_balance, {:address_coin_balance, coin_balance}] diff --git a/apps/indexer/test/indexer/fetcher/token_balance_test.exs b/apps/indexer/test/indexer/fetcher/token_balance_test.exs index 0c4fb0be44f8..0aac5a82608e 100644 --- a/apps/indexer/test/indexer/fetcher/token_balance_test.exs +++ b/apps/indexer/test/indexer/fetcher/token_balance_test.exs @@ -368,4 +368,48 @@ defmodule Indexer.Fetcher.TokenBalanceTest do assert TokenBalance.import_token_balances(token_balances_params) == :ok end end + + describe "native token and coin balance sync" do + test "updating native token balance updates coin balance" do + # Setup: Insert a token and address, and set NATIVE_TOKEN_ADDRESS env + contract = insert(:token) + address = insert(:address) + block = insert(:block, number: 12345) + native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" + System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) + + # Insert a token balance for the native token + token_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block.number, + token_contract_address_hash: native_token_address, + token_id: nil, + value: 42_000_000, + token_type: "ERC-20" + } + ] + + # Call the import function (or the function that triggers the sync) + assert :ok = Indexer.Fetcher.TokenBalance.import_token_balances(token_balance_params) + + # Assert token balance exists + token_balance = + Explorer.Chain.Address.TokenBalance + |> where([tb], tb.address_hash == ^address.hash and tb.token_contract_address_hash == ^native_token_address) + |> Repo.one() + + assert token_balance.value == Decimal.new(42_000_000) + + # Assert coin balance was also updated (adjust schema/module as needed) + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash) + |> Repo.one() + IO.inspect(coin_balance.value, label: "Wei struct") + IO.inspect(Map.from_struct(coin_balance.value), label: "Wei struct fields") + assert Map.values(Map.from_struct(coin_balance.value)) |> hd() == Decimal.new(42_000_000) + end + end + end From 8dd346484db764fcaddd5f4c366ca6dc42bb6abd Mon Sep 17 00:00:00 2001 From: banasa44 Date: Mon, 21 Apr 2025 19:44:20 +0000 Subject: [PATCH 09/17] feat(indexer): add test for updating balances when a coin transfer happens. --- .../fetcher/coin_balance/catchup_test.exs | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/apps/indexer/test/indexer/fetcher/coin_balance/catchup_test.exs b/apps/indexer/test/indexer/fetcher/coin_balance/catchup_test.exs index b330dccef57d..875f4abc0455 100644 --- a/apps/indexer/test/indexer/fetcher/coin_balance/catchup_test.exs +++ b/apps/indexer/test/indexer/fetcher/coin_balance/catchup_test.exs @@ -525,4 +525,62 @@ defmodule Indexer.Fetcher.CoinBalance.CatchupTest do } } end + + describe "import_fetched_balances/2" do + test "importing native coin balance also creates native token balance" do + # Set up the native token address as in your env + native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" + System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) + + # Insert an address and block + address = insert(:address) + block_number = block_number() + block_quantity = integer_to_quantity(block_number) + res = eth_block_number_fake_response(block_quantity) + # Prepare coin balance params as would be fetched + coin_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block_number, + value: Decimal.new(123_456_789), + value_fetched_at: DateTime.utc_now() + } + ] + + + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_getBlockByNumber", + params: [^block_quantity, true] + } + ], _options -> + {:ok, [res]} # Use your eth_block_number_fake_response/1 helper + end) + + # Call the import_fetched_balances function + result = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(coin_balance_params) + assert match?({:ok, _}, result) + + # Assert coin balance exists + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block_number) + |> Repo.one() + + assert coin_balance.value == %Explorer.Chain.Wei{value: Decimal.new(123_456_789)} + + # Assert token balance for the native token was also created + token_balance = + Explorer.Chain.Address.TokenBalance + |> where([tb], tb.address_hash == ^address.hash and tb.block_number == ^block_number and tb.token_contract_address_hash == ^native_token_address) + |> Repo.one() + + assert token_balance.value == Decimal.new(123_456_789) + assert token_balance.token_type == "ERC-20" + end + end + end From f821b677019049c873517137451ec1a96639af7d Mon Sep 17 00:00:00 2001 From: banasa44 Date: Tue, 22 Apr 2025 08:52:54 +0000 Subject: [PATCH 10/17] refactor(indexer/test): move xrpl-evm specific test in a separate file. --- .../fetcher/coin_balance/catchup_test.exs | 58 ------- .../indexer/fetcher/token_balance_test.exs | 44 ----- .../test/indexer/fetcher/xrpl_evm_test.exs | 154 ++++++++++++++++++ 3 files changed, 154 insertions(+), 102 deletions(-) create mode 100644 apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs diff --git a/apps/indexer/test/indexer/fetcher/coin_balance/catchup_test.exs b/apps/indexer/test/indexer/fetcher/coin_balance/catchup_test.exs index 875f4abc0455..b330dccef57d 100644 --- a/apps/indexer/test/indexer/fetcher/coin_balance/catchup_test.exs +++ b/apps/indexer/test/indexer/fetcher/coin_balance/catchup_test.exs @@ -525,62 +525,4 @@ defmodule Indexer.Fetcher.CoinBalance.CatchupTest do } } end - - describe "import_fetched_balances/2" do - test "importing native coin balance also creates native token balance" do - # Set up the native token address as in your env - native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" - System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) - - # Insert an address and block - address = insert(:address) - block_number = block_number() - block_quantity = integer_to_quantity(block_number) - res = eth_block_number_fake_response(block_quantity) - # Prepare coin balance params as would be fetched - coin_balance_params = [ - %{ - address_hash: to_string(address.hash), - block_number: block_number, - value: Decimal.new(123_456_789), - value_fetched_at: DateTime.utc_now() - } - ] - - - EthereumJSONRPC.Mox - |> expect(:json_rpc, fn [ - %{ - id: 0, - jsonrpc: "2.0", - method: "eth_getBlockByNumber", - params: [^block_quantity, true] - } - ], _options -> - {:ok, [res]} # Use your eth_block_number_fake_response/1 helper - end) - - # Call the import_fetched_balances function - result = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(coin_balance_params) - assert match?({:ok, _}, result) - - # Assert coin balance exists - coin_balance = - Explorer.Chain.Address.CoinBalance - |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block_number) - |> Repo.one() - - assert coin_balance.value == %Explorer.Chain.Wei{value: Decimal.new(123_456_789)} - - # Assert token balance for the native token was also created - token_balance = - Explorer.Chain.Address.TokenBalance - |> where([tb], tb.address_hash == ^address.hash and tb.block_number == ^block_number and tb.token_contract_address_hash == ^native_token_address) - |> Repo.one() - - assert token_balance.value == Decimal.new(123_456_789) - assert token_balance.token_type == "ERC-20" - end - end - end diff --git a/apps/indexer/test/indexer/fetcher/token_balance_test.exs b/apps/indexer/test/indexer/fetcher/token_balance_test.exs index 0aac5a82608e..0c4fb0be44f8 100644 --- a/apps/indexer/test/indexer/fetcher/token_balance_test.exs +++ b/apps/indexer/test/indexer/fetcher/token_balance_test.exs @@ -368,48 +368,4 @@ defmodule Indexer.Fetcher.TokenBalanceTest do assert TokenBalance.import_token_balances(token_balances_params) == :ok end end - - describe "native token and coin balance sync" do - test "updating native token balance updates coin balance" do - # Setup: Insert a token and address, and set NATIVE_TOKEN_ADDRESS env - contract = insert(:token) - address = insert(:address) - block = insert(:block, number: 12345) - native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" - System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) - - # Insert a token balance for the native token - token_balance_params = [ - %{ - address_hash: to_string(address.hash), - block_number: block.number, - token_contract_address_hash: native_token_address, - token_id: nil, - value: 42_000_000, - token_type: "ERC-20" - } - ] - - # Call the import function (or the function that triggers the sync) - assert :ok = Indexer.Fetcher.TokenBalance.import_token_balances(token_balance_params) - - # Assert token balance exists - token_balance = - Explorer.Chain.Address.TokenBalance - |> where([tb], tb.address_hash == ^address.hash and tb.token_contract_address_hash == ^native_token_address) - |> Repo.one() - - assert token_balance.value == Decimal.new(42_000_000) - - # Assert coin balance was also updated (adjust schema/module as needed) - coin_balance = - Explorer.Chain.Address.CoinBalance - |> where([cb], cb.address_hash == ^address.hash) - |> Repo.one() - IO.inspect(coin_balance.value, label: "Wei struct") - IO.inspect(Map.from_struct(coin_balance.value), label: "Wei struct fields") - assert Map.values(Map.from_struct(coin_balance.value)) |> hd() == Decimal.new(42_000_000) - end - end - end diff --git a/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs b/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs new file mode 100644 index 000000000000..be2d8e76a0b6 --- /dev/null +++ b/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs @@ -0,0 +1,154 @@ +defmodule Indexer.XRPLEVM.IntegrationTest do + use EthereumJSONRPC.Case, async: false + use Explorer.DataCase + + import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1] + import Mox + + alias Explorer.Chain.{Address, Hash, Wei} + alias Explorer.Chain.Cache.BlockNumber + alias Indexer.Fetcher.CoinBalance.Catchup, as: CoinBalanceCatchup + + alias Explorer.Repo + + @moduletag :capture_log + + setup :verify_on_exit! + setup :set_mox_global + + setup do + start_supervised!({Task.Supervisor, name: Indexer.TaskSupervisor}) + + initial_config = Application.get_env(:explorer, Explorer.Chain.Cache.BlockNumber) + Application.put_env(:explorer, Explorer.Chain.Cache.BlockNumber, enabled: true) + + on_exit(fn -> + Application.put_env(:explorer, Explorer.Chain.Cache.BlockNumber, initial_config) + end) + + :ok + end + + defp eth_block_number_fake_response(block_quantity) do + %{ + id: 0, + jsonrpc: "2.0", + result: %{ + "author" => "0x0000000000000000000000000000000000000000", + "difficulty" => "0x20000", + "extraData" => "0x", + "gasLimit" => "0x663be0", + "gasUsed" => "0x0", + "hash" => "0x5b28c1bfd3a15230c9a46b399cd0f9a6920d432e85381cc6a140b06e8410112f", + "logsBloom" => "...", + "miner" => "0x0000000000000000000000000000000000000000", + "number" => block_quantity, + "parentHash" => "0x0000000000000000000000000000000000000000000000000000000000000000", + "receiptsRoot" => "...", + "sealFields" => ["0x80", "..."], + "sha3Uncles" => "...", + "signature" => "...", + "size" => "0x215", + "stateRoot" => "...", + "step" => "0", + "timestamp" => "0x0", + "totalDifficulty" => "0x20000", + "transactions" => [], + "transactionsRoot" => "...", + "uncles" => [] + } + } + end + + test "updating native token balance updates coin balance" do + address = insert(:address) + block = insert(:block, number: 12345) + native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" + System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) + + # Insert a token balance for the native token + token_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block.number, + token_contract_address_hash: native_token_address, + token_id: nil, + value: 42_000_000, + token_type: "ERC-20" + } + ] + + # Call the import function (or the function that triggers the sync) + assert :ok = Indexer.Fetcher.TokenBalance.import_token_balances(token_balance_params) + + # Assert token balance exists + token_balance = + Explorer.Chain.Address.TokenBalance + |> where([tb], tb.address_hash == ^address.hash and tb.token_contract_address_hash == ^native_token_address) + |> Repo.one() + + assert token_balance.value == Decimal.new(42_000_000) + + # Assert coin balance was also updated + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash) + |> Repo.one() + + assert coin_balance.value == %Wei{value: Decimal.new(42_000_000)} end + + test "importing native coin balance also creates native token balance" do + # Set up the native token address as in your env + native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" + System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) + + # Insert an address and block + address = insert(:address) + block_number = block_number() + block_quantity = integer_to_quantity(block_number) + res = eth_block_number_fake_response(block_quantity) + # Prepare coin balance params as would be fetched + coin_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block_number, + value: Decimal.new(123_456_789), + value_fetched_at: DateTime.utc_now() + } + ] + + + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_getBlockByNumber", + params: [^block_quantity, true] + } + ], _options -> + {:ok, [res]} # Use your eth_block_number_fake_response/1 helper + end) + + # Call the import_fetched_balances function + result = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(coin_balance_params) + assert match?({:ok, _}, result) + + # Assert coin balance exists + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block_number) + |> Repo.one() + + assert coin_balance.value == %Explorer.Chain.Wei{value: Decimal.new(123_456_789)} + + # Assert token balance for the native token was also created + token_balance = + Explorer.Chain.Address.TokenBalance + |> where([tb], tb.address_hash == ^address.hash and tb.block_number == ^block_number and tb.token_contract_address_hash == ^native_token_address) + |> Repo.one() + + assert token_balance.value == Decimal.new(123_456_789) + assert token_balance.token_type == "ERC-20" + end +end From a01a38375f54ddad4f2d1efbab91cc1588d4a4f2 Mon Sep 17 00:00:00 2001 From: banasa44 Date: Tue, 22 Apr 2025 09:05:58 +0000 Subject: [PATCH 11/17] feat(indexer): add negative test for not updating balances when a NON native_token transfer happens. --- .../test/indexer/fetcher/xrpl_evm_test.exs | 139 ++++++++++-------- 1 file changed, 81 insertions(+), 58 deletions(-) diff --git a/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs b/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs index be2d8e76a0b6..53575878486d 100644 --- a/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs +++ b/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs @@ -66,7 +66,6 @@ defmodule Indexer.XRPLEVM.IntegrationTest do native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) - # Insert a token balance for the native token token_balance_params = [ %{ address_hash: to_string(address.hash), @@ -78,10 +77,8 @@ defmodule Indexer.XRPLEVM.IntegrationTest do } ] - # Call the import function (or the function that triggers the sync) assert :ok = Indexer.Fetcher.TokenBalance.import_token_balances(token_balance_params) - # Assert token balance exists token_balance = Explorer.Chain.Address.TokenBalance |> where([tb], tb.address_hash == ^address.hash and tb.token_contract_address_hash == ^native_token_address) @@ -89,66 +86,92 @@ defmodule Indexer.XRPLEVM.IntegrationTest do assert token_balance.value == Decimal.new(42_000_000) - # Assert coin balance was also updated coin_balance = Explorer.Chain.Address.CoinBalance |> where([cb], cb.address_hash == ^address.hash) |> Repo.one() - assert coin_balance.value == %Wei{value: Decimal.new(42_000_000)} end + assert coin_balance.value == %Wei{value: Decimal.new(42_000_000)} + end test "importing native coin balance also creates native token balance" do - # Set up the native token address as in your env - native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" - System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) - - # Insert an address and block - address = insert(:address) - block_number = block_number() - block_quantity = integer_to_quantity(block_number) - res = eth_block_number_fake_response(block_quantity) - # Prepare coin balance params as would be fetched - coin_balance_params = [ - %{ - address_hash: to_string(address.hash), - block_number: block_number, - value: Decimal.new(123_456_789), - value_fetched_at: DateTime.utc_now() - } - ] - - - EthereumJSONRPC.Mox - |> expect(:json_rpc, fn [ - %{ - id: 0, - jsonrpc: "2.0", - method: "eth_getBlockByNumber", - params: [^block_quantity, true] - } - ], _options -> - {:ok, [res]} # Use your eth_block_number_fake_response/1 helper - end) - - # Call the import_fetched_balances function - result = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(coin_balance_params) - assert match?({:ok, _}, result) - - # Assert coin balance exists - coin_balance = - Explorer.Chain.Address.CoinBalance - |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block_number) - |> Repo.one() - - assert coin_balance.value == %Explorer.Chain.Wei{value: Decimal.new(123_456_789)} - - # Assert token balance for the native token was also created - token_balance = - Explorer.Chain.Address.TokenBalance - |> where([tb], tb.address_hash == ^address.hash and tb.block_number == ^block_number and tb.token_contract_address_hash == ^native_token_address) - |> Repo.one() - - assert token_balance.value == Decimal.new(123_456_789) - assert token_balance.token_type == "ERC-20" - end + native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" + System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) + + address = insert(:address) + block_number = block_number() + block_quantity = integer_to_quantity(block_number) + res = eth_block_number_fake_response(block_quantity) + + coin_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block_number, + value: Decimal.new(123_456_789), + value_fetched_at: DateTime.utc_now() + } + ] + + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_getBlockByNumber", + params: [^block_quantity, true] + } + ], + _options -> + {:ok, [res]} + end) + + result = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(coin_balance_params) + assert match?({:ok, _}, result) + + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block_number) + |> Repo.one() + + assert coin_balance.value == %Explorer.Chain.Wei{value: Decimal.new(123_456_789)} + + token_balance = + Explorer.Chain.Address.TokenBalance + |> where( + [tb], + tb.address_hash == ^address.hash and tb.block_number == ^block_number and + tb.token_contract_address_hash == ^native_token_address + ) + |> Repo.one() + + assert token_balance.value == Decimal.new(123_456_789) + assert token_balance.token_type == "ERC-20" + end + + test "importing non-native token balance does not update coin balance" do + address = insert(:address) + block = insert(:block, number: 12345) + non_native_token_address = "0x1234567890abcdef1234567890abcdef12345678" + System.put_env("NATIVE_TOKEN_ADDRESS", "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE") + + token_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block.number, + token_contract_address_hash: non_native_token_address, + token_id: nil, + value: 42_000_000, + token_type: "ERC-20" + } + ] + + assert :ok = Indexer.Fetcher.TokenBalance.import_token_balances(token_balance_params) + + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block.number) + |> Repo.one() + + assert is_nil(coin_balance) + end end From 8bfe5d28174c8abb202e069379fc7c3bba5fdcaa Mon Sep 17 00:00:00 2001 From: banasa44 Date: Tue, 22 Apr 2025 09:07:23 +0000 Subject: [PATCH 12/17] chore(indexer): run mix format (elixir's prettier) --- .../indexer/fetcher/coin_balance/helper.ex | 6 -- .../lib/indexer/fetcher/token_balance.ex | 77 ++++++++++--------- .../transform/address_coin_balances.ex | 2 - .../xrpl-evm/token_to_coin_balance.ex | 3 +- 4 files changed, 41 insertions(+), 47 deletions(-) diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex index ead693b4dd7c..15837a0c1140 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex @@ -15,7 +15,6 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do alias Indexer.TokenBalances alias Indexer.Transformers.CoinToTokenBalanceTransformer - @doc false # credo:disable-for-next-line Credo.Check.Design.DuplicatedCode def child_spec([init_options, gen_server_options], defaults, module) do @@ -96,8 +95,6 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do end) end - - def import_fetched_balances(params_list, broadcast_type \\ false) do value_fetched_at = DateTime.utc_now() @@ -106,7 +103,6 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do importable_balances_daily_params = balances_daily_params(params_list, json_rpc_named_arguments) addresses_params = balances_params_to_address_params(importable_balances_params) - # Transform coin balances to also produce token balances for the native token transformed_params = CoinToTokenBalanceTransformer.transform_address_coin_balances(importable_balances_params) {final_coin_balances, token_balance_entries} = @@ -130,8 +126,6 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do }) end - - def import_fetched_daily_balances(params_list, broadcast_type \\ false) do value_fetched_at = DateTime.utc_now() diff --git a/apps/indexer/lib/indexer/fetcher/token_balance.ex b/apps/indexer/lib/indexer/fetcher/token_balance.ex index 0f770d6ed393..b8fa791c9964 100644 --- a/apps/indexer/lib/indexer/fetcher/token_balance.ex +++ b/apps/indexer/lib/indexer/fetcher/token_balance.ex @@ -206,47 +206,50 @@ defmodule Indexer.Fetcher.TokenBalance do Timex.shift(Timex.now(), seconds: value) end -def import_token_balances(token_balances_params) do - # Existing formatting and filtering steps - addresses_params = format_and_filter_address_params(token_balances_params) - formatted_token_balances_params = format_and_filter_token_balance_params(token_balances_params) - - # Apply transformer on the formatted token balances to get potential coin balance entries. - transformed_params = - Indexer.Transformers.TokenToCoinBalanceTransformer.transform_address_token_balances(formatted_token_balances_params) - - # Separate into token balance entries and coin balance entries. - {final_token_balances, coin_balance_entries} = - Enum.split_with(transformed_params, fn - {:address_coin_balance, _} -> false - _ -> true - end) + def import_token_balances(token_balances_params) do + # Existing formatting and filtering steps + addresses_params = format_and_filter_address_params(token_balances_params) + formatted_token_balances_params = format_and_filter_token_balance_params(token_balances_params) + + # Apply transformer on the formatted token balances to get potential coin balance entries. + transformed_params = + Indexer.Transformers.TokenToCoinBalanceTransformer.transform_address_token_balances( + formatted_token_balances_params + ) - # Process current token balances as before, using output from the transformer. - final_current_token_balances = TokenBalances.to_address_current_token_balances(final_token_balances) - - # Build import params, now including coin balances. - import_params = %{ - addresses: %{params: addresses_params}, - address_token_balances: %{params: final_token_balances}, - address_current_token_balances: %{params: final_current_token_balances}, - address_coin_balances: - %{params: Enum.map(coin_balance_entries, fn {:address_coin_balance, coin_balance} -> coin_balance end)}, - timeout: @timeout - } - - case Chain.import(import_params) do - {:ok, _} -> - :ok + # Separate into token balance entries and coin balance entries. + {final_token_balances, coin_balance_entries} = + Enum.split_with(transformed_params, fn + {:address_coin_balance, _} -> false + _ -> true + end) - {:error, reason} -> - Logger.debug(fn -> ["failed to import token balances: ", inspect(reason)] end, - error_count: Enum.count(token_balances_params) - ) + # Process current token balances as before, using output from the transformer. + final_current_token_balances = TokenBalances.to_address_current_token_balances(final_token_balances) + + # Build import params, now including coin balances. + import_params = %{ + addresses: %{params: addresses_params}, + address_token_balances: %{params: final_token_balances}, + address_current_token_balances: %{params: final_current_token_balances}, + address_coin_balances: %{ + params: Enum.map(coin_balance_entries, fn {:address_coin_balance, coin_balance} -> coin_balance end) + }, + timeout: @timeout + } + + case Chain.import(import_params) do + {:ok, _} -> + :ok + + {:error, reason} -> + Logger.debug(fn -> ["failed to import token balances: ", inspect(reason)] end, + error_count: Enum.count(token_balances_params) + ) - :error + :error + end end -end defp format_and_filter_address_params(token_balances_params) do token_balances_params diff --git a/apps/indexer/lib/indexer/transform/address_coin_balances.ex b/apps/indexer/lib/indexer/transform/address_coin_balances.ex index 5bab3ccb80d8..3787261d552f 100644 --- a/apps/indexer/lib/indexer/transform/address_coin_balances.ex +++ b/apps/indexer/lib/indexer/transform/address_coin_balances.ex @@ -6,7 +6,6 @@ defmodule Indexer.Transform.AddressCoinBalances do alias Explorer.Chain.TokenTransfer - def params_set(%{} = import_options) do Enum.reduce(import_options, MapSet.new(), &reducer/2) end @@ -30,7 +29,6 @@ defmodule Indexer.Transform.AddressCoinBalances do defp reducer({:logs_params, logs_params}, acc) when is_list(logs_params) do # a log MUST have address_hash and block_number logs_params - |> Enum.reject( &(&1.first_topic == TokenTransfer.constant() or &1.first_topic == TokenTransfer.erc1155_single_transfer_signature() or diff --git a/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex index bdad9efdf5ce..b2f04701686e 100644 --- a/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex +++ b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex @@ -1,5 +1,4 @@ defmodule Indexer.Transformers.TokenToCoinBalanceTransformer do - @doc """ Transforms changes to `address_token_balances` into changes for `address_coin_balances` if the token contract address hash matches the hardcoded value. @@ -9,7 +8,7 @@ defmodule Indexer.Transformers.TokenToCoinBalanceTransformer do Enum.flat_map(params, fn token_balance -> if token_balance[:token_contract_address_hash] == native_token_address do - # Create a corresponding address_coin_balance entry + coin_balance = %{ address_hash: token_balance[:address_hash], value: token_balance[:value], From 1158a454744dd6ef26755f2106cfa1553471d13b Mon Sep 17 00:00:00 2001 From: banasa44 Date: Tue, 22 Apr 2025 13:36:16 +0000 Subject: [PATCH 13/17] feat(indexer): add address_coin_balances_daily to import_token_balances --- .../indexer/fetcher/coin_balance/helper.ex | 2 +- .../lib/indexer/fetcher/token_balance.ex | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex index 15837a0c1140..0322ae6e0096 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex @@ -219,7 +219,7 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do end) end - defp balances_daily_params(params_list, json_rpc_named_arguments) do + def balances_daily_params(params_list, json_rpc_named_arguments) do block_timestamp_map = block_timestamp_map(params_list, json_rpc_named_arguments) params_list diff --git a/apps/indexer/lib/indexer/fetcher/token_balance.ex b/apps/indexer/lib/indexer/fetcher/token_balance.ex index b8fa791c9964..af333cd4abf1 100644 --- a/apps/indexer/lib/indexer/fetcher/token_balance.ex +++ b/apps/indexer/lib/indexer/fetcher/token_balance.ex @@ -25,6 +25,8 @@ defmodule Indexer.Fetcher.TokenBalance do alias Indexer.{BufferedTask, TokenBalances, Tracer} alias Indexer.Fetcher.TokenBalance.Supervisor, as: TokenBalanceSupervisor + import Indexer.Fetcher.CoinBalance.Helper, only: [balances_daily_params: 2] + @behaviour BufferedTask @default_max_batch_size 100 @@ -227,14 +229,19 @@ defmodule Indexer.Fetcher.TokenBalance do # Process current token balances as before, using output from the transformer. final_current_token_balances = TokenBalances.to_address_current_token_balances(final_token_balances) + coin_balance_params = + Enum.map(coin_balance_entries, fn {:address_coin_balance, coin_balance} -> coin_balance end) + + json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments) + importable_balances_daily_params = balances_daily_params(coin_balance_params, json_rpc_named_arguments) + # Build import params, now including coin balances. import_params = %{ - addresses: %{params: addresses_params}, + addresses: %{params: addresses_params, with: :balance_changeset}, address_token_balances: %{params: final_token_balances}, address_current_token_balances: %{params: final_current_token_balances}, - address_coin_balances: %{ - params: Enum.map(coin_balance_entries, fn {:address_coin_balance, coin_balance} -> coin_balance end) - }, + address_coin_balances: %{params: coin_balance_params}, + address_coin_balances_daily: %{params: importable_balances_daily_params}, timeout: @timeout } @@ -253,8 +260,12 @@ defmodule Indexer.Fetcher.TokenBalance do defp format_and_filter_address_params(token_balances_params) do token_balances_params - |> Enum.map(&%{hash: &1.address_hash}) - |> Enum.uniq() + |> Enum.group_by(& &1.address_hash) + |> Map.values() + |> Stream.map(&Enum.max_by(&1, fn %{block_number: block_number} -> block_number end)) + |> Enum.map(fn %{address_hash: address_hash, block_number: block_number, value: value} -> + %{hash: address_hash, fetched_coin_balance_block_number: block_number, fetched_coin_balance: value} + end) end defp format_and_filter_token_balance_params(token_balances_params) do From b2b2d1eb662ca04294b018328893c66a8ca5e88f Mon Sep 17 00:00:00 2001 From: banasa44 Date: Tue, 22 Apr 2025 13:36:44 +0000 Subject: [PATCH 14/17] feat(indexer): add test for address_coin_balances_daily in import_token_balances --- .../test/indexer/fetcher/xrpl_evm_test.exs | 334 +++++++++++++----- 1 file changed, 237 insertions(+), 97 deletions(-) diff --git a/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs b/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs index 53575878486d..913fdc9b4835 100644 --- a/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs +++ b/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs @@ -2,12 +2,10 @@ defmodule Indexer.XRPLEVM.IntegrationTest do use EthereumJSONRPC.Case, async: false use Explorer.DataCase - import EthereumJSONRPC, only: [integer_to_quantity: 1, quantity_to_integer: 1] + import EthereumJSONRPC, only: [integer_to_quantity: 1] import Mox - alias Explorer.Chain.{Address, Hash, Wei} - alias Explorer.Chain.Cache.BlockNumber - alias Indexer.Fetcher.CoinBalance.Catchup, as: CoinBalanceCatchup + alias Explorer.Chain.{Address, Wei} alias Explorer.Repo @@ -26,7 +24,21 @@ defmodule Indexer.XRPLEVM.IntegrationTest do Application.put_env(:explorer, Explorer.Chain.Cache.BlockNumber, initial_config) end) - :ok + native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" + System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) + address = insert(:address) + other_address = insert(:address) + block_number = block_number() + block_quantity = integer_to_quantity(block_number) + res = eth_block_number_fake_response(block_quantity) + + {:ok, + native_token_address: native_token_address, + address: address, + other_address: other_address, + block_number: block_number, + block_quantity: block_quantity, + res: res} end defp eth_block_number_fake_response(block_quantity) do @@ -60,118 +72,246 @@ defmodule Indexer.XRPLEVM.IntegrationTest do } end - test "updating native token balance updates coin balance" do - address = insert(:address) - block = insert(:block, number: 12345) - native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" - System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) + describe "import_token_balances/1 (token→coin)" do + test "updating native token balance updates coin balance", %{ + native_token_address: native_token_address, + address: address, + block_number: block_number, + block_quantity: block_quantity, + res: res + } do + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_getBlockByNumber", + params: [^block_quantity, true] + } + ], + _options -> + {:ok, [res]} + end) - token_balance_params = [ - %{ - address_hash: to_string(address.hash), - block_number: block.number, - token_contract_address_hash: native_token_address, - token_id: nil, - value: 42_000_000, - token_type: "ERC-20" - } - ] + token_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block_number, + token_contract_address_hash: native_token_address, + token_id: nil, + value: 42_000_000, + token_type: "ERC-20" + } + ] + + assert :ok = Indexer.Fetcher.TokenBalance.import_token_balances(token_balance_params) + + token_balance = + Explorer.Chain.Address.TokenBalance + |> where([tb], tb.address_hash == ^address.hash and tb.token_contract_address_hash == ^native_token_address) + |> Repo.one() + + assert token_balance.value == Decimal.new(42_000_000) - assert :ok = Indexer.Fetcher.TokenBalance.import_token_balances(token_balance_params) + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash) + |> Repo.one() - token_balance = - Explorer.Chain.Address.TokenBalance - |> where([tb], tb.address_hash == ^address.hash and tb.token_contract_address_hash == ^native_token_address) - |> Repo.one() + assert coin_balance.value == %Wei{value: Decimal.new(42_000_000)} - assert token_balance.value == Decimal.new(42_000_000) + addr = Repo.get!(Address, address.hash) + assert addr.fetched_coin_balance == %Wei{value: Decimal.new(42_000_000)} + assert addr.fetched_coin_balance_block_number == block_number - coin_balance = - Explorer.Chain.Address.CoinBalance - |> where([cb], cb.address_hash == ^address.hash) - |> Repo.one() + daily = + Explorer.Chain.Address.CoinBalanceDaily + |> where([d], d.address_hash == ^address.hash and d.value == ^Decimal.new(42_000_000)) + |> Repo.one() - assert coin_balance.value == %Wei{value: Decimal.new(42_000_000)} + assert daily.day == ~D[1970-01-01] + + + end + + test "importing non-native token balance does not update coin balance", %{ + address: address, + other_address: other_address, + block_number: block_number + } do + token_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block_number, + token_contract_address_hash: other_address.hash, + token_id: nil, + value: 42_000_000, + token_type: "ERC-20" + } + ] + + assert :ok = Indexer.Fetcher.TokenBalance.import_token_balances(token_balance_params) + + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block_number) + |> Repo.one() + + assert is_nil(coin_balance) + end end - test "importing native coin balance also creates native token balance" do - native_token_address = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" - System.put_env("NATIVE_TOKEN_ADDRESS", native_token_address) + describe "import_fetched_balances/2 (coin→token)" do + test "importing native coin balance also creates native token balance", %{ + native_token_address: native_token_address, + address: address, + other_address: other_address, + block_number: block_number, + block_quantity: block_quantity, + res: res + } do + coin_balance_params = [ + %{ + address_hash: to_string(address.hash), + block_number: block_number, + value: Decimal.new(123_456_789), + value_fetched_at: DateTime.utc_now() + } + ] - address = insert(:address) - block_number = block_number() - block_quantity = integer_to_quantity(block_number) - res = eth_block_number_fake_response(block_quantity) + EthereumJSONRPC.Mox + |> expect(:json_rpc, fn [ + %{ + id: 0, + jsonrpc: "2.0", + method: "eth_getBlockByNumber", + params: [^block_quantity, true] + } + ], + _options -> + {:ok, [res]} + end) - coin_balance_params = [ - %{ - address_hash: to_string(address.hash), - block_number: block_number, - value: Decimal.new(123_456_789), - value_fetched_at: DateTime.utc_now() - } - ] + result = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(coin_balance_params) + assert match?({:ok, _}, result) - EthereumJSONRPC.Mox - |> expect(:json_rpc, fn [ + coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block_number) + |> Repo.one() + + assert coin_balance.value == %Explorer.Chain.Wei{value: Decimal.new(123_456_789)} + + other_coin_balance = + Explorer.Chain.Address.CoinBalance + |> where([cb], cb.address_hash == ^other_address.hash) + |> Repo.one() + + assert is_nil(other_coin_balance) + + token_balance = + Explorer.Chain.Address.TokenBalance + |> where( + [tb], + tb.address_hash == ^address.hash and tb.block_number == ^block_number and + tb.token_contract_address_hash == ^native_token_address + ) + |> Repo.one() + + assert token_balance.value == Decimal.new(123_456_789) + assert token_balance.token_type == "ERC-20" + + current = + Explorer.Chain.Address.CurrentTokenBalance + |> where( + [ctb], + ctb.address_hash == ^address.hash and + ctb.token_contract_address_hash == ^native_token_address + ) + |> Repo.one() + + assert current.value == Decimal.new(123_456_789) + + + addr = Repo.get!(Address, address.hash) + assert addr.fetched_coin_balance == %Wei{value: Decimal.new(123_456_789)} + assert addr.fetched_coin_balance_block_number == block_number + end + + test "latest block snapshot wins", %{ + native_token_address: native_token_address, + address: address, + block_number: block_number + } do + b1 = block_number + b2 = block_number + 1 + q1 = integer_to_quantity(b1) + q2 = integer_to_quantity(b2) + res1 = eth_block_number_fake_response(q1) + res2 = eth_block_number_fake_response(q2) + + EthereumJSONRPC.Mox + |> stub(:json_rpc, fn + [%{method: "eth_getBlockByNumber", params: [^q1, true]}], _opts -> {:ok, [res1]} + [%{method: "eth_getBlockByNumber", params: [^q2, true]}], _opts -> {:ok, [res2]} + end) + + params = [ + %{ + address_hash: to_string(address.hash), + block_number: b1, + value: Decimal.new(100), + value_fetched_at: DateTime.utc_now() + }, + %{ + address_hash: to_string(address.hash), + block_number: b2, + value: Decimal.new(200), + value_fetched_at: DateTime.utc_now() + } + ] + + assert {:ok, _} = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(params) + + current = + Explorer.Chain.Address.CurrentTokenBalance + |> where([ctb], ctb.address_hash == ^address.hash and ctb.token_contract_address_hash == ^native_token_address) + |> Repo.one() + + assert current.value == Decimal.new(200) + end + + test "idempotent coin import doesn't duplicate rows", %{ + address: address, + block_number: block_number + } do + EthereumJSONRPC.Mox + |> stub(:json_rpc, fn [ %{ id: 0, jsonrpc: "2.0", method: "eth_getBlockByNumber", - params: [^block_quantity, true] + params: [block_quantity, true] } ], _options -> - {:ok, [res]} - end) - - result = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(coin_balance_params) - assert match?({:ok, _}, result) - - coin_balance = - Explorer.Chain.Address.CoinBalance - |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block_number) - |> Repo.one() - - assert coin_balance.value == %Explorer.Chain.Wei{value: Decimal.new(123_456_789)} - - token_balance = - Explorer.Chain.Address.TokenBalance - |> where( - [tb], - tb.address_hash == ^address.hash and tb.block_number == ^block_number and - tb.token_contract_address_hash == ^native_token_address - ) - |> Repo.one() - - assert token_balance.value == Decimal.new(123_456_789) - assert token_balance.token_type == "ERC-20" - end - - test "importing non-native token balance does not update coin balance" do - address = insert(:address) - block = insert(:block, number: 12345) - non_native_token_address = "0x1234567890abcdef1234567890abcdef12345678" - System.put_env("NATIVE_TOKEN_ADDRESS", "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE") - - token_balance_params = [ - %{ - address_hash: to_string(address.hash), - block_number: block.number, - token_contract_address_hash: non_native_token_address, - token_id: nil, - value: 42_000_000, - token_type: "ERC-20" - } - ] + {:ok, [eth_block_number_fake_response(block_quantity)]} + end) - assert :ok = Indexer.Fetcher.TokenBalance.import_token_balances(token_balance_params) + params = [ + %{ + address_hash: to_string(address.hash), + block_number: block_number, + value: Decimal.new(50), + value_fetched_at: DateTime.utc_now() + } + ] - coin_balance = - Explorer.Chain.Address.CoinBalance - |> where([cb], cb.address_hash == ^address.hash and cb.block_number == ^block.number) - |> Repo.one() + assert {:ok, _} = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(params) + assert {:ok, _} = Indexer.Fetcher.CoinBalance.Helper.import_fetched_balances(params) - assert is_nil(coin_balance) + assert Repo.aggregate(Explorer.Chain.Address.CoinBalance, :count, :block_number) == 1 + assert Repo.aggregate(Explorer.Chain.Address.CoinBalance, :count, :address_hash) == 1 + end end end From 00d60c9ce10a582ddc05f23c7d13809905ae3a42 Mon Sep 17 00:00:00 2001 From: banasa44 Date: Tue, 22 Apr 2025 13:42:27 +0000 Subject: [PATCH 15/17] chore: mix format --- apps/indexer/lib/indexer/fetcher/token_balance.ex | 5 ----- .../indexer/transform/xrpl-evm/token_to_coin_balance.ex | 1 - apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs | 9 +++------ 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/apps/indexer/lib/indexer/fetcher/token_balance.ex b/apps/indexer/lib/indexer/fetcher/token_balance.ex index af333cd4abf1..836902804392 100644 --- a/apps/indexer/lib/indexer/fetcher/token_balance.ex +++ b/apps/indexer/lib/indexer/fetcher/token_balance.ex @@ -209,24 +209,20 @@ defmodule Indexer.Fetcher.TokenBalance do end def import_token_balances(token_balances_params) do - # Existing formatting and filtering steps addresses_params = format_and_filter_address_params(token_balances_params) formatted_token_balances_params = format_and_filter_token_balance_params(token_balances_params) - # Apply transformer on the formatted token balances to get potential coin balance entries. transformed_params = Indexer.Transformers.TokenToCoinBalanceTransformer.transform_address_token_balances( formatted_token_balances_params ) - # Separate into token balance entries and coin balance entries. {final_token_balances, coin_balance_entries} = Enum.split_with(transformed_params, fn {:address_coin_balance, _} -> false _ -> true end) - # Process current token balances as before, using output from the transformer. final_current_token_balances = TokenBalances.to_address_current_token_balances(final_token_balances) coin_balance_params = @@ -235,7 +231,6 @@ defmodule Indexer.Fetcher.TokenBalance do json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments) importable_balances_daily_params = balances_daily_params(coin_balance_params, json_rpc_named_arguments) - # Build import params, now including coin balances. import_params = %{ addresses: %{params: addresses_params, with: :balance_changeset}, address_token_balances: %{params: final_token_balances}, diff --git a/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex index b2f04701686e..37e6568d8107 100644 --- a/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex +++ b/apps/indexer/lib/indexer/transform/xrpl-evm/token_to_coin_balance.ex @@ -8,7 +8,6 @@ defmodule Indexer.Transformers.TokenToCoinBalanceTransformer do Enum.flat_map(params, fn token_balance -> if token_balance[:token_contract_address_hash] == native_token_address do - coin_balance = %{ address_hash: token_balance[:address_hash], value: token_balance[:value], diff --git a/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs b/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs index 913fdc9b4835..b1ed6ae98974 100644 --- a/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs +++ b/apps/indexer/test/indexer/fetcher/xrpl_evm_test.exs @@ -130,8 +130,6 @@ defmodule Indexer.XRPLEVM.IntegrationTest do |> Repo.one() assert daily.day == ~D[1970-01-01] - - end test "importing non-native token balance does not update coin balance", %{ @@ -232,10 +230,9 @@ defmodule Indexer.XRPLEVM.IntegrationTest do assert current.value == Decimal.new(123_456_789) - - addr = Repo.get!(Address, address.hash) - assert addr.fetched_coin_balance == %Wei{value: Decimal.new(123_456_789)} - assert addr.fetched_coin_balance_block_number == block_number + addr = Repo.get!(Address, address.hash) + assert addr.fetched_coin_balance == %Wei{value: Decimal.new(123_456_789)} + assert addr.fetched_coin_balance_block_number == block_number end test "latest block snapshot wins", %{ From 2e47f497006beb7aadeec73db2122f67e0f6c472 Mon Sep 17 00:00:00 2001 From: banasa44 Date: Tue, 22 Apr 2025 13:58:34 +0000 Subject: [PATCH 16/17] chore(indexer): variable renaming --- .../lib/indexer/fetcher/coin_balance/helper.ex | 8 ++++---- apps/indexer/lib/indexer/fetcher/token_balance.ex | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex index 0322ae6e0096..a9944a0e797a 100644 --- a/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex +++ b/apps/indexer/lib/indexer/fetcher/coin_balance/helper.ex @@ -105,7 +105,7 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do transformed_params = CoinToTokenBalanceTransformer.transform_address_coin_balances(importable_balances_params) - {final_coin_balances, token_balance_entries} = + {coin_balance_entries, token_balance_entries} = Enum.split_with(transformed_params, fn {:address_token_balance, _} -> false _ -> true @@ -114,14 +114,14 @@ defmodule Indexer.Fetcher.CoinBalance.Helper do token_balance_params = Enum.map(token_balance_entries, fn {:address_token_balance, token_balance} -> token_balance end) - current_token_balance_params = TokenBalances.to_address_current_token_balances(token_balance_params) + current_token_balance_entries = TokenBalances.to_address_current_token_balances(token_balance_params) Chain.import(%{ addresses: %{params: addresses_params, with: :balance_changeset}, - address_coin_balances: %{params: final_coin_balances}, + address_coin_balances: %{params: coin_balance_entries}, address_coin_balances_daily: %{params: importable_balances_daily_params}, address_token_balances: %{params: token_balance_params}, - address_current_token_balances: %{params: current_token_balance_params}, + address_current_token_balances: %{params: current_token_balance_entries}, broadcast: broadcast_type }) end diff --git a/apps/indexer/lib/indexer/fetcher/token_balance.ex b/apps/indexer/lib/indexer/fetcher/token_balance.ex index 836902804392..719c57bfffa1 100644 --- a/apps/indexer/lib/indexer/fetcher/token_balance.ex +++ b/apps/indexer/lib/indexer/fetcher/token_balance.ex @@ -217,26 +217,26 @@ defmodule Indexer.Fetcher.TokenBalance do formatted_token_balances_params ) - {final_token_balances, coin_balance_entries} = + {token_balance_entries, coin_balance_entries} = Enum.split_with(transformed_params, fn {:address_coin_balance, _} -> false _ -> true end) - final_current_token_balances = TokenBalances.to_address_current_token_balances(final_token_balances) + current_token_balances = TokenBalances.to_address_current_token_balances(token_balance_entries) coin_balance_params = Enum.map(coin_balance_entries, fn {:address_coin_balance, coin_balance} -> coin_balance end) json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments) - importable_balances_daily_params = balances_daily_params(coin_balance_params, json_rpc_named_arguments) + daily_coin_balance_entries = balances_daily_params(coin_balance_params, json_rpc_named_arguments) import_params = %{ addresses: %{params: addresses_params, with: :balance_changeset}, - address_token_balances: %{params: final_token_balances}, - address_current_token_balances: %{params: final_current_token_balances}, + address_token_balances: %{params: token_balance_entries}, + address_current_token_balances: %{params: current_token_balances}, address_coin_balances: %{params: coin_balance_params}, - address_coin_balances_daily: %{params: importable_balances_daily_params}, + address_coin_balances_daily: %{params: daily_coin_balance_entries}, timeout: @timeout } From 2a70df5a608ce4b37e379cdfb24ff00ed56b5d29 Mon Sep 17 00:00:00 2001 From: banasa44 Date: Tue, 22 Apr 2025 14:15:21 +0000 Subject: [PATCH 17/17] chore --- .devcontainer/bin/bs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/bin/bs b/.devcontainer/bin/bs index 90900b07d879..145c152531dc 100755 --- a/.devcontainer/bin/bs +++ b/.devcontainer/bin/bs @@ -203,7 +203,7 @@ initialize_db() { # Define the compile subroutine compile() { - MIX_DEBUG=1 mix compile + mix compile } # Define the recompile subroutine