From cd32d3fd1114c2d9d08e9715a1d1ba748052a0f9 Mon Sep 17 00:00:00 2001 From: "Odysseas.eth" Date: Fri, 16 Jan 2026 13:54:45 +0100 Subject: [PATCH 1/7] Use tx gas params for backtest replay --- scripts/backtesting/transaction_fetcher.sh | 34 +++++++++++++++------- src/CredibleTestWithBacktesting.sol | 24 ++++++++++++++- src/utils/BacktestingTypes.sol | 3 ++ src/utils/BacktestingUtils.sol | 34 ++++++++++++++++++---- 4 files changed, 78 insertions(+), 17 deletions(-) diff --git a/scripts/backtesting/transaction_fetcher.sh b/scripts/backtesting/transaction_fetcher.sh index 59ba145..18cc129 100755 --- a/scripts/backtesting/transaction_fetcher.sh +++ b/scripts/backtesting/transaction_fetcher.sh @@ -76,7 +76,7 @@ PERFORMANCE: Balance these based on your RPC provider's rate limits. OUTPUT: - simple: count|hash|from|to|value|data|blockNumber|txIndex|gasPrice|... + simple: count|hash|from|to|value|data|blockNumber|txIndex|gasPrice|gasLimit|maxFeePerGas|maxPriorityFeePerGas|... json: Array of transaction objects with labeled fields EOF @@ -282,7 +282,10 @@ fetch_transactions_trace_filter() { local tx_to=$(echo "$tx_data" | jq -r '.to // empty') local tx_value=$(echo "$tx_data" | jq -r '.value') local tx_input=$(echo "$tx_data" | jq -r '.input') - local tx_gas_price=$(echo "$tx_data" | jq -r '.gasPrice') + local tx_gas_price=$(echo "$tx_data" | jq -r '.gasPrice // "0x0"') + local tx_gas_limit=$(echo "$tx_data" | jq -r '.gas // "0x0"') + local tx_max_fee_per_gas=$(echo "$tx_data" | jq -r '.maxFeePerGas // "0x0"') + local tx_max_priority_fee_per_gas=$(echo "$tx_data" | jq -r '.maxPriorityFeePerGas // "0x0"') # Check if transaction succeeded on-chain local receipt_request=$(jq -n \ @@ -305,7 +308,7 @@ fetch_transactions_trace_filter() { # Only output transaction if it succeeded (status == "0x1") if [[ "$tx_status" == "0x1" ]]; then - echo "$tx_hash|$tx_from|$tx_to|$tx_value|$tx_input|$block_num|$tx_index|$tx_gas_price" >> "$output_file" + echo "$tx_hash|$tx_from|$tx_to|$tx_value|$tx_input|$block_num|$tx_index|$tx_gas_price|$tx_gas_limit|$tx_max_fee_per_gas|$tx_max_priority_fee_per_gas" >> "$output_file" ((tx_count++)) fi fi @@ -432,13 +435,16 @@ fetch_block_transactions() { local tx_value=$(echo "$tx" | jq -r '.value') local tx_input=$(echo "$tx" | jq -r '.input') local tx_index_hex=$(echo "$tx" | jq -r '.transactionIndex') - local tx_gas_price=$(echo "$tx" | jq -r '.gasPrice') + local tx_gas_price=$(echo "$tx" | jq -r '.gasPrice // "0x0"') + local tx_gas_limit=$(echo "$tx" | jq -r '.gas // "0x0"') + local tx_max_fee_per_gas=$(echo "$tx" | jq -r '.maxFeePerGas // "0x0"') + local tx_max_priority_fee_per_gas=$(echo "$tx" | jq -r '.maxPriorityFeePerGas // "0x0"') # Convert transaction index to decimal local tx_index_decimal=$(hex_to_decimal "$tx_index_hex") - # Output transaction in the format: hash|from|to|value|data|blockNumber|txIndex|gasPrice - echo "$tx_hash|$tx_from|$tx_to|$tx_value|$tx_input|$block_num_decimal|$tx_index_decimal|$tx_gas_price" >> "$output_file" + # Output transaction in the format: hash|from|to|value|data|blockNumber|txIndex|gasPrice|gasLimit|maxFeePerGas|maxPriorityFeePerGas + echo "$tx_hash|$tx_from|$tx_to|$tx_value|$tx_input|$block_num_decimal|$tx_index_decimal|$tx_gas_price|$tx_gas_limit|$tx_max_fee_per_gas|$tx_max_priority_fee_per_gas" >> "$output_file" fi fi fi @@ -522,7 +528,7 @@ format_transactions() { # Convert to JSON format echo "[" local first=true - while IFS='|' read -r hash from to value data block_number tx_index gas_price; do + while IFS='|' read -r hash from to value data block_number tx_index gas_price gas_limit max_fee_per_gas max_priority_fee_per_gas; do if [[ "$first" == "true" ]]; then first=false else @@ -537,6 +543,9 @@ format_transactions() { --arg block_number "$block_number" \ --arg tx_index "$tx_index" \ --arg gas_price "$gas_price" \ + --arg gas_limit "$gas_limit" \ + --arg max_fee_per_gas "$max_fee_per_gas" \ + --arg max_priority_fee_per_gas "$max_priority_fee_per_gas" \ '{ hash: $hash, from: $from, @@ -545,17 +554,20 @@ format_transactions() { data: $data, block_number: $block_number, transaction_index: $tx_index, - gas_price: $gas_price + gas_price: $gas_price, + gas_limit: $gas_limit, + max_fee_per_gas: $max_fee_per_gas, + max_priority_fee_per_gas: $max_priority_fee_per_gas }' | tr -d '\n' done < "$all_transactions_file" echo echo "]" ;; *) - # Simple format: count|hash|from|to|value|data|blockNumber|txIndex|gasPrice|... + # Simple format: count|hash|from|to|value|data|blockNumber|txIndex|gasPrice|gasLimit|maxFeePerGas|maxPriorityFeePerGas|... echo -n "$tx_count" - while IFS='|' read -r hash from to value data block_number tx_index gas_price; do - echo -n "|$hash|$from|$to|$value|$data|$block_number|$tx_index|$gas_price" + while IFS='|' read -r hash from to value data block_number tx_index gas_price gas_limit max_fee_per_gas max_priority_fee_per_gas; do + echo -n "|$hash|$from|$to|$value|$data|$block_number|$tx_index|$gas_price|$gas_limit|$max_fee_per_gas|$max_priority_fee_per_gas" done < "$all_transactions_file" ;; esac diff --git a/src/CredibleTestWithBacktesting.sol b/src/CredibleTestWithBacktesting.sol index ed856f8..a90b504 100644 --- a/src/CredibleTestWithBacktesting.sol +++ b/src/CredibleTestWithBacktesting.sol @@ -217,8 +217,23 @@ abstract contract CredibleTestWithBacktesting is CredibleTest, Test { cl.assertion({adopter: targetContract, createData: assertionCreationCode, fnSelector: assertionSelector}); // Execute the transaction + uint256 gasPrice = txData.gasPrice; + if (txData.maxFeePerGas > 0) { + gasPrice = txData.maxFeePerGas; + } + if (gasPrice > 0) { + vm.txGasPrice(gasPrice); + } + vm.prank(txData.from, txData.from); - (bool callSuccess, bytes memory returnData) = txData.to.call{value: txData.value}(txData.data); + bool callSuccess; + bytes memory returnData; + if (txData.gasLimit > 0) { + (callSuccess, returnData) = + txData.to.call{value: txData.value, gas: txData.gasLimit}(txData.data); + } else { + (callSuccess, returnData) = txData.to.call{value: txData.value}(txData.data); + } console.log(string.concat("Transaction status: ", callSuccess ? "Success" : "Failure")); if (callSuccess) { @@ -239,6 +254,13 @@ abstract contract CredibleTestWithBacktesting is CredibleTest, Test { validation.result = BacktestingTypes.ValidationResult.Skipped; validation.errorMessage = "Function selector not triggered by this transaction"; validation.isProtocolViolation = false; + } else if ( + BacktestingUtils.startsWith(revertReason, "Assertion Executor Error: ForkTxExecutionError") + ) { + // Replay failed before assertion execution (e.g., insufficient funds for max fee) + validation.result = BacktestingTypes.ValidationResult.ReplayFailure; + validation.errorMessage = revertReason; + validation.isProtocolViolation = false; } else { // Actual assertion failure (protocol violation) validation.result = BacktestingTypes.ValidationResult.AssertionFailed; diff --git a/src/utils/BacktestingTypes.sol b/src/utils/BacktestingTypes.sol index c2d45de..626f020 100644 --- a/src/utils/BacktestingTypes.sol +++ b/src/utils/BacktestingTypes.sol @@ -23,6 +23,9 @@ library BacktestingTypes { uint256 blockNumber; uint256 transactionIndex; uint256 gasPrice; + uint256 gasLimit; + uint256 maxFeePerGas; + uint256 maxPriorityFeePerGas; } /// @notice Detailed validation result with error information diff --git a/src/utils/BacktestingUtils.sol b/src/utils/BacktestingUtils.sol index 1a8e7bd..63c0721 100644 --- a/src/utils/BacktestingUtils.sol +++ b/src/utils/BacktestingUtils.sol @@ -144,14 +144,35 @@ library BacktestingUtils { transactions = new BacktestingTypes.TransactionData[](count); - // Each transaction has 8 fields: hash|from|to|value|data|block|txIndex|gasPrice - uint256 fieldsPerTransaction = 8; - uint256 expectedParts = 1 + (count * fieldsPerTransaction); // +1 for count at beginning - require(parts.length >= expectedParts, "Insufficient transaction data"); + // Each transaction has 8 fields (legacy) or 11 fields (extended): + // hash|from|to|value|data|block|txIndex|gasPrice[|gasLimit|maxFeePerGas|maxPriorityFeePerGas] + uint256 legacyFieldsPerTransaction = 8; + uint256 extendedFieldsPerTransaction = 11; + uint256 legacyExpectedParts = 1 + (count * legacyFieldsPerTransaction); // +1 for count at beginning + uint256 extendedExpectedParts = 1 + (count * extendedFieldsPerTransaction); + uint256 fieldsPerTransaction = parts.length >= extendedExpectedParts + ? extendedFieldsPerTransaction + : legacyFieldsPerTransaction; + + require( + parts.length >= legacyExpectedParts, + "Insufficient transaction data" + ); for (uint256 i = 0; i < count; i++) { uint256 startIndex = 1 + (i * fieldsPerTransaction); // Skip count at beginning + uint256 gasPrice = stringToUint(parts[startIndex + 7]); + uint256 gasLimit = 0; + uint256 maxFeePerGas = 0; + uint256 maxPriorityFeePerGas = 0; + + if (fieldsPerTransaction == extendedFieldsPerTransaction) { + gasLimit = stringToUint(parts[startIndex + 8]); + maxFeePerGas = stringToUint(parts[startIndex + 9]); + maxPriorityFeePerGas = stringToUint(parts[startIndex + 10]); + } + transactions[i] = BacktestingTypes.TransactionData({ hash: stringToBytes32(parts[startIndex]), from: stringToAddress(parts[startIndex + 1]), @@ -160,7 +181,10 @@ library BacktestingUtils { data: hexStringToBytes(parts[startIndex + 4]), blockNumber: stringToUint(parts[startIndex + 5]), transactionIndex: stringToUint(parts[startIndex + 6]), - gasPrice: stringToUint(parts[startIndex + 7]) + gasPrice: gasPrice, + gasLimit: gasLimit, + maxFeePerGas: maxFeePerGas, + maxPriorityFeePerGas: maxPriorityFeePerGas }); } } From 402573b12d1cee1aef74593bf0a49271c6918cc8 Mon Sep 17 00:00:00 2001 From: "Odysseas.eth" Date: Fri, 16 Jan 2026 15:03:43 +0100 Subject: [PATCH 2/7] Backtesting: ensure sender covers max fee --- src/CredibleTestWithBacktesting.sol | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/CredibleTestWithBacktesting.sol b/src/CredibleTestWithBacktesting.sol index a90b504..d294b5c 100644 --- a/src/CredibleTestWithBacktesting.sol +++ b/src/CredibleTestWithBacktesting.sol @@ -225,6 +225,17 @@ abstract contract CredibleTestWithBacktesting is CredibleTest, Test { vm.txGasPrice(gasPrice); } + // Ensure the sender can cover max-fee checks even when the tx gas limit defaults to 2^24. + uint256 feeGasPrice = gasPrice > 0 ? gasPrice : block.basefee; + if (feeGasPrice > 0) { + uint256 defaultGasLimit = 1 << 24; + uint256 effectiveGasLimit = txData.gasLimit > defaultGasLimit ? txData.gasLimit : defaultGasLimit; + uint256 minBalance = (effectiveGasLimit * feeGasPrice) + txData.value; + if (txData.from.balance < minBalance) { + vm.deal(txData.from, minBalance); + } + } + vm.prank(txData.from, txData.from); bool callSuccess; bytes memory returnData; From af928c966cf34c4ad40383f553a5d76d99e48206 Mon Sep 17 00:00:00 2001 From: "Odysseas.eth" Date: Fri, 16 Jan 2026 15:08:39 +0100 Subject: [PATCH 3/7] Backtesting: cap basefee for replay affordability --- src/CredibleTestWithBacktesting.sol | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/CredibleTestWithBacktesting.sol b/src/CredibleTestWithBacktesting.sol index d294b5c..79f2053 100644 --- a/src/CredibleTestWithBacktesting.sol +++ b/src/CredibleTestWithBacktesting.sol @@ -225,15 +225,12 @@ abstract contract CredibleTestWithBacktesting is CredibleTest, Test { vm.txGasPrice(gasPrice); } - // Ensure the sender can cover max-fee checks even when the tx gas limit defaults to 2^24. - uint256 feeGasPrice = gasPrice > 0 ? gasPrice : block.basefee; - if (feeGasPrice > 0) { - uint256 defaultGasLimit = 1 << 24; - uint256 effectiveGasLimit = txData.gasLimit > defaultGasLimit ? txData.gasLimit : defaultGasLimit; - uint256 minBalance = (effectiveGasLimit * feeGasPrice) + txData.value; - if (txData.from.balance < minBalance) { - vm.deal(txData.from, minBalance); - } + // Cap basefee to avoid fork replay failures when the tx gas limit defaults to 2^24. + uint256 defaultGasLimit = 1 << 24; + uint256 effectiveGasLimit = txData.gasLimit > defaultGasLimit ? txData.gasLimit : defaultGasLimit; + uint256 maxAffordableBasefee = txData.from.balance / effectiveGasLimit; + if (block.basefee > maxAffordableBasefee) { + vm.fee(maxAffordableBasefee); } vm.prank(txData.from, txData.from); From 5c393a753c1ccc140756688bc4644d5c4ac3ad40 Mon Sep 17 00:00:00 2001 From: "Odysseas.eth" Date: Fri, 16 Jan 2026 16:59:02 +0100 Subject: [PATCH 4/7] Always fork by tx hash for backtests --- src/CredibleTestWithBacktesting.sol | 9 ++++----- src/utils/BacktestingTypes.sol | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/CredibleTestWithBacktesting.sol b/src/CredibleTestWithBacktesting.sol index 79f2053..917bae0 100644 --- a/src/CredibleTestWithBacktesting.sol +++ b/src/CredibleTestWithBacktesting.sol @@ -203,12 +203,11 @@ abstract contract CredibleTestWithBacktesting is CredibleTest, Test { BacktestingTypes.TransactionData memory txData, bool forkByTxHash ) private returns (BacktestingTypes.ValidationDetails memory validation) { - // Fork at the transaction's block, or by tx hash if requested - if (forkByTxHash) { - vm.createSelectFork(rpcUrl, txData.hash); - } else { - vm.createSelectFork(rpcUrl, txData.blockNumber); + // Always fork by tx hash to ensure pre-transaction state; block forks are post-state. + if (!forkByTxHash) { + // Keep the flag for compatibility, but avoid unsafe post-state replays. } + vm.createSelectFork(rpcUrl, txData.hash); // Prepare transaction sender vm.stopPrank(); diff --git a/src/utils/BacktestingTypes.sol b/src/utils/BacktestingTypes.sol index 626f020..54204bb 100644 --- a/src/utils/BacktestingTypes.sol +++ b/src/utils/BacktestingTypes.sol @@ -45,7 +45,7 @@ library BacktestingTypes { string rpcUrl; bool detailedBlocks; // Enable detailed block summaries in output bool useTraceFilter; // Use trace_filter (fast) instead of debug_traceTransaction (slow) - bool forkByTxHash; // Fork by transaction hash instead of block number (more accurate but slower) + bool forkByTxHash; // Fork by transaction hash for correct pre-tx state; block forks are unsafe. } /// @notice Enhanced backtesting results with detailed categorization From d2b9cd84c16ce5757fb8e5177d850dcce38663ab Mon Sep 17 00:00:00 2001 From: "Odysseas.eth" Date: Fri, 16 Jan 2026 17:08:09 +0100 Subject: [PATCH 5/7] Format backtesting sources --- src/CredibleTestWithBacktesting.sol | 7 ++----- src/utils/BacktestingUtils.sol | 12 ++++-------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/CredibleTestWithBacktesting.sol b/src/CredibleTestWithBacktesting.sol index 917bae0..93554cc 100644 --- a/src/CredibleTestWithBacktesting.sol +++ b/src/CredibleTestWithBacktesting.sol @@ -236,8 +236,7 @@ abstract contract CredibleTestWithBacktesting is CredibleTest, Test { bool callSuccess; bytes memory returnData; if (txData.gasLimit > 0) { - (callSuccess, returnData) = - txData.to.call{value: txData.value, gas: txData.gasLimit}(txData.data); + (callSuccess, returnData) = txData.to.call{value: txData.value, gas: txData.gasLimit}(txData.data); } else { (callSuccess, returnData) = txData.to.call{value: txData.value}(txData.data); } @@ -261,9 +260,7 @@ abstract contract CredibleTestWithBacktesting is CredibleTest, Test { validation.result = BacktestingTypes.ValidationResult.Skipped; validation.errorMessage = "Function selector not triggered by this transaction"; validation.isProtocolViolation = false; - } else if ( - BacktestingUtils.startsWith(revertReason, "Assertion Executor Error: ForkTxExecutionError") - ) { + } else if (BacktestingUtils.startsWith(revertReason, "Assertion Executor Error: ForkTxExecutionError")) { // Replay failed before assertion execution (e.g., insufficient funds for max fee) validation.result = BacktestingTypes.ValidationResult.ReplayFailure; validation.errorMessage = revertReason; diff --git a/src/utils/BacktestingUtils.sol b/src/utils/BacktestingUtils.sol index 63c0721..e724dd0 100644 --- a/src/utils/BacktestingUtils.sol +++ b/src/utils/BacktestingUtils.sol @@ -150,14 +150,10 @@ library BacktestingUtils { uint256 extendedFieldsPerTransaction = 11; uint256 legacyExpectedParts = 1 + (count * legacyFieldsPerTransaction); // +1 for count at beginning uint256 extendedExpectedParts = 1 + (count * extendedFieldsPerTransaction); - uint256 fieldsPerTransaction = parts.length >= extendedExpectedParts - ? extendedFieldsPerTransaction - : legacyFieldsPerTransaction; - - require( - parts.length >= legacyExpectedParts, - "Insufficient transaction data" - ); + uint256 fieldsPerTransaction = + parts.length >= extendedExpectedParts ? extendedFieldsPerTransaction : legacyFieldsPerTransaction; + + require(parts.length >= legacyExpectedParts, "Insufficient transaction data"); for (uint256 i = 0; i < count; i++) { uint256 startIndex = 1 + (i * fieldsPerTransaction); // Skip count at beginning From 6f18c82a72430faef20f5e40b038f9ef4b64d263 Mon Sep 17 00:00:00 2001 From: "Odysseas.eth" Date: Sat, 17 Jan 2026 02:07:20 +0100 Subject: [PATCH 6/7] Improve backtesting trace support --- scripts/backtesting/trace_support_harness.sh | 198 ++++++++ scripts/backtesting/transaction_fetcher.sh | 447 +++++++++++++++---- src/CredibleTestWithBacktesting.sol | 6 + 3 files changed, 565 insertions(+), 86 deletions(-) create mode 100755 scripts/backtesting/trace_support_harness.sh diff --git a/scripts/backtesting/trace_support_harness.sh b/scripts/backtesting/trace_support_harness.sh new file mode 100755 index 0000000..d50ad1b --- /dev/null +++ b/scripts/backtesting/trace_support_harness.sh @@ -0,0 +1,198 @@ +#!/bin/bash + +# Trace Support Harness +# Checks which trace APIs are supported by an RPC endpoint. + +set -eo pipefail + +RPC_URL="" +BLOCK_NUMBER="" +TARGET_CONTRACT="" + +usage() { + printf '%s\n' \ + "Usage: $0 --rpc-url URL --block NUMBER [--target-contract ADDRESS]" \ + "" \ + "Checks trace API support on a given RPC." \ + "" \ + "Options:" \ + " --rpc-url URL RPC endpoint URL (required)" \ + " --block NUMBER Block number to probe (required)" \ + " --target-contract ADDRESS Contract address for trace_filter test (optional)" \ + " -h, --help Show this help message" \ + "" \ + "Example:" \ + " $0 --rpc-url https://your.rpc --block 23717632 --target-contract 0xBA12222222228d8Ba445958a75a0704d566BF2C8" +} + +check_dependencies() { + local missing=() + command -v curl >/dev/null 2>&1 || missing+=("curl") + command -v jq >/dev/null 2>&1 || missing+=("jq") + + if [[ ${#missing[@]} -gt 0 ]]; then + echo "Error: Missing required tools: ${missing[*]}" >&2 + exit 1 + fi +} + +is_method_unsupported() { + local response="$1" + local error_code + local error_msg + error_code=$(echo "$response" | jq -r '.error.code // empty' 2>/dev/null) + error_msg=$(echo "$response" | jq -r '.error.message // empty' 2>/dev/null | tr '[:upper:]' '[:lower:]') + + if [[ "$error_code" == "-32601" ]]; then + return 0 + fi + if [[ -n "$error_msg" ]]; then + if [[ "$error_msg" == *"method not found"* ]] || + [[ "$error_msg" == *"does not exist"* ]] || + [[ "$error_msg" == *"not available"* ]] || + [[ "$error_msg" == *"unknown method"* ]] || + [[ "$error_msg" == *"not supported"* ]]; then + return 0 + fi + fi + return 1 +} + +# Parse args +while [[ $# -gt 0 ]]; do + case $1 in + --rpc-url) + RPC_URL="$2" + shift 2 + ;; + --block) + BLOCK_NUMBER="$2" + shift 2 + ;; + --target-contract) + TARGET_CONTRACT="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Error: Unknown option $1" >&2 + usage + exit 1 + ;; + esac +done + +if [[ -z "$RPC_URL" || -z "$BLOCK_NUMBER" ]]; then + echo "Error: --rpc-url and --block are required" >&2 + usage + exit 1 +fi + +if ! [[ "$BLOCK_NUMBER" =~ ^[0-9]+$ ]]; then + echo "Error: Block number must be a positive integer" >&2 + exit 1 +fi + +check_dependencies + +BLOCK_HEX=$(printf "0x%x" "$BLOCK_NUMBER") + +printf "Trace support check for block %s\n" "$BLOCK_NUMBER" + +# trace_filter (requires target) +if [[ -n "$TARGET_CONTRACT" ]]; then + TRACE_FILTER_REQ=$(jq -n \ + --arg block_hex "$BLOCK_HEX" \ + --arg target "$TARGET_CONTRACT" \ + '{"jsonrpc":"2.0","method":"trace_filter","params":[{"fromBlock":$block_hex,"toBlock":$block_hex,"toAddress":[$target]}],"id":1}') + + TRACE_FILTER_RESP=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "$TRACE_FILTER_REQ" \ + --max-time 30 \ + "$RPC_URL") + + if echo "$TRACE_FILTER_RESP" | jq -e '.error' >/dev/null 2>&1; then + if is_method_unsupported "$TRACE_FILTER_RESP"; then + echo "trace_filter: unsupported" + else + echo "trace_filter: error (see response for details)" + fi + else + echo "trace_filter: supported" + fi +else + echo "trace_filter: skipped (no --target-contract provided)" +fi + +# debug_traceBlockByNumber +TRACE_BLOCK_REQ=$(jq -n \ + --arg block_hex "$BLOCK_HEX" \ + '{"jsonrpc":"2.0","method":"debug_traceBlockByNumber","params":[$block_hex,{"tracer":"callTracer"}],"id":1}') + +TRACE_BLOCK_RESP=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "$TRACE_BLOCK_REQ" \ + --max-time 30 \ + "$RPC_URL") + +if echo "$TRACE_BLOCK_RESP" | jq -e '.error' >/dev/null 2>&1; then + if is_method_unsupported "$TRACE_BLOCK_RESP"; then + echo "debug_traceBlockByNumber: unsupported" + else + echo "debug_traceBlockByNumber: error (see response for details)" + fi +else + echo "debug_traceBlockByNumber: supported" +fi + +# debug_traceTransaction (needs a tx hash) +TX_HASH="" +for ((i=0; i<5; i++)); do + local_block=$((BLOCK_NUMBER - i)) + if [[ $local_block -lt 0 ]]; then + break + fi + local_block_hex=$(printf "0x%x" "$local_block") + BLOCK_REQ=$(jq -n \ + --arg block_hex "$local_block_hex" \ + '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":[$block_hex,false],"id":1}') + BLOCK_RESP=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "$BLOCK_REQ" \ + --max-time 30 \ + "$RPC_URL") + TX_HASH=$(echo "$BLOCK_RESP" | jq -r '.result.transactions[0] // empty') + if [[ -n "$TX_HASH" ]]; then + break + fi + TX_HASH="" +done + +if [[ -z "$TX_HASH" ]]; then + echo "debug_traceTransaction: skipped (no tx hash found in recent blocks)" + exit 0 +fi + +TRACE_TX_REQ=$(jq -n \ + --arg tx_hash "$TX_HASH" \ + '{"jsonrpc":"2.0","method":"debug_traceTransaction","params":[$tx_hash,{"tracer":"callTracer"}],"id":1}') + +TRACE_TX_RESP=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "$TRACE_TX_REQ" \ + --max-time 30 \ + "$RPC_URL") + +if echo "$TRACE_TX_RESP" | jq -e '.error' >/dev/null 2>&1; then + if is_method_unsupported "$TRACE_TX_RESP"; then + echo "debug_traceTransaction: unsupported" + else + echo "debug_traceTransaction: error (see response for details)" + fi +else + echo "debug_traceTransaction: supported" +fi diff --git a/scripts/backtesting/transaction_fetcher.sh b/scripts/backtesting/transaction_fetcher.sh index 18cc129..594068b 100755 --- a/scripts/backtesting/transaction_fetcher.sh +++ b/scripts/backtesting/transaction_fetcher.sh @@ -2,7 +2,7 @@ # Transaction Fetcher - Bash Implementation # Fetches blockchain transactions for backtesting -# Uses transaction receipts (logs) to detect internal calls +# Uses trace APIs (trace_filter or debug_trace*) to detect internal calls set -eo pipefail @@ -13,6 +13,7 @@ MAX_CONCURRENT=5 DETAILED_BLOCKS=false USE_TRACE_FILTER=false TRACE_FILTER_BATCH_SIZE=100 +TRACE_METHOD="" TEMP_DIR="" START_TIME="" @@ -49,7 +50,7 @@ OPTIONS: --output-format FORMAT Output format: simple or json (default: simple) --batch-size SIZE Batch size for processing (default: 10) --max-concurrent COUNT Maximum concurrent requests (default: 5) - --use-trace-filter Use trace_filter for fast internal call detection (default: false) + --use-trace-filter Use traces for internal call detection (trace_filter w/ debug_trace* fallback) --trace-filter-batch-size SIZE Batch size for trace_filter (default: 100) --detailed-blocks Enable detailed per-block summaries (default: false) -h, --help Show this help message @@ -61,7 +62,7 @@ EXAMPLES: --start-block 10000000 \\ --end-block 10000100 - # Use trace_filter for fast internal call detection + # Use trace-based detection (trace_filter or debug_trace* fallback) $0 --rpc-url \$MAINNET_RPC_URL \\ --target-contract 0xBA12222222228d8Ba445958a75a0704d566BF2C8 \\ --start-block 23717632 \\ @@ -184,6 +185,132 @@ count_rpc_calls() { fi } +# Check if RPC error indicates unsupported method +is_method_unsupported() { + local response="$1" + local error_code + local error_msg + error_code=$(echo "$response" | jq -r '.error.code // empty' 2>/dev/null) + error_msg=$(echo "$response" | jq -r '.error.message // empty' 2>/dev/null | tr '[:upper:]' '[:lower:]') + + if [[ "$error_code" == "-32601" ]]; then + return 0 + fi + if [[ -n "$error_msg" ]]; then + if [[ "$error_msg" == *"method not found"* ]] || + [[ "$error_msg" == *"does not exist"* ]] || + [[ "$error_msg" == *"not available"* ]] || + [[ "$error_msg" == *"unknown method"* ]] || + [[ "$error_msg" == *"not supported"* ]]; then + return 0 + fi + fi + return 1 +} + +# Fetch transaction details and receipts for a list of tx hashes +emit_transactions_from_hashes() { + local rpc_url="$1" + local tx_hashes="$2" + local output_file="$3" + + local tx_count=0 + local tx_processed=0 + + if [[ -n "$tx_hashes" ]]; then + while IFS= read -r tx_hash; do + [[ -z "$tx_hash" ]] && continue + + # Add small delay every 5 transactions to avoid rate limiting + if [[ $((tx_processed % 5)) -eq 0 && $tx_processed -gt 0 ]]; then + sleep 0.1 + fi + ((tx_processed++)) + + # Fetch actual transaction data using eth_getTransactionByHash + local tx_request + tx_request=$(jq -n \ + --arg tx_hash "$tx_hash" \ + '{ + "jsonrpc": "2.0", + "method": "eth_getTransactionByHash", + "params": [$tx_hash], + "id": 1 + }') + + echo "1" >> "$RPC_COUNTER_DIR/tx_fetch.count" + local tx_response + tx_response=$(retry_with_backoff 5 curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "$tx_request" \ + --max-time 30 \ + "$rpc_url") + + local tx_data + tx_data=$(echo "$tx_response" | jq -r '.result') + + if [[ -n "$tx_data" && "$tx_data" != "null" ]]; then + local tx_from + local tx_to + local tx_value + local tx_input + local tx_gas_price + local tx_gas_limit + local tx_max_fee_per_gas + local tx_max_priority_fee_per_gas + local tx_block_num_hex + local tx_index_hex + + tx_from=$(echo "$tx_data" | jq -r '.from') + tx_to=$(echo "$tx_data" | jq -r '.to // empty') + tx_value=$(echo "$tx_data" | jq -r '.value') + tx_input=$(echo "$tx_data" | jq -r '.input') + tx_gas_price=$(echo "$tx_data" | jq -r '.gasPrice // "0x0"') + tx_gas_limit=$(echo "$tx_data" | jq -r '.gas // "0x0"') + tx_max_fee_per_gas=$(echo "$tx_data" | jq -r '.maxFeePerGas // "0x0"') + tx_max_priority_fee_per_gas=$(echo "$tx_data" | jq -r '.maxPriorityFeePerGas // "0x0"') + tx_block_num_hex=$(echo "$tx_data" | jq -r '.blockNumber // empty') + tx_index_hex=$(echo "$tx_data" | jq -r '.transactionIndex // empty') + + local block_num + local tx_index + block_num=$(hex_to_decimal "$tx_block_num_hex") + tx_index=$(hex_to_decimal "$tx_index_hex") + + # Check if transaction succeeded on-chain + local receipt_request + receipt_request=$(jq -n \ + --arg tx_hash "$tx_hash" \ + '{ + "jsonrpc": "2.0", + "method": "eth_getTransactionReceipt", + "params": [$tx_hash], + "id": 1 + }') + + echo "1" >> "$RPC_COUNTER_DIR/receipt_fetch.count" + local receipt_response + receipt_response=$(retry_with_backoff 5 curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "$receipt_request" \ + --max-time 30 \ + "$rpc_url") + + local tx_status + tx_status=$(echo "$receipt_response" | jq -r '.result.status // empty') + + # Only output transaction if it succeeded (status == "0x1") + if [[ "$tx_status" == "0x1" ]]; then + echo "$tx_hash|$tx_from|$tx_to|$tx_value|$tx_input|$block_num|$tx_index|$tx_gas_price|$tx_gas_limit|$tx_max_fee_per_gas|$tx_max_priority_fee_per_gas" >> "$output_file" + ((tx_count++)) + fi + fi + done <<< "$tx_hashes" + fi + + echo "$tx_count" +} + # Fetch transactions using trace_filter (much faster for internal calls) fetch_transactions_trace_filter() { local rpc_url="$1" @@ -225,96 +352,206 @@ fetch_transactions_trace_filter() { # Check for errors if [[ -z "$trace_response" ]] || echo "$trace_response" | jq -e '.error' > /dev/null 2>&1; then - local error_msg=$(echo "$trace_response" | jq -r '.error.message // "Unknown error"') + local error_msg + error_msg=$(echo "$trace_response" | jq -r '.error.message // "Unknown error"') echo "Error: trace_filter failed: $error_msg" >&2 + if is_method_unsupported "$trace_response"; then + echo "trace_filter unsupported on this RPC, falling back to debug_trace*" >&2 + echo "0" + return 2 + fi + echo "0" return 1 fi - # Parse traces and group by transaction # Extract unique transaction hashes that involve the target contract - # We'll fetch full transaction data separately to avoid issues with internal calls - local tx_hashes=$(echo "$trace_response" | jq -r ' - .result - | group_by(.transactionHash) - | map(.[0]) - | map({ - hash: .transactionHash, - blockNumber: .blockNumber, - transactionPosition: .transactionPosition - }) - | .[] - | [.hash, (.blockNumber | tostring), (.transactionPosition | tostring)] | join("|") - ') - - # Fetch full transaction data for each transaction hash and filter by receipt status - local tx_count=0 - local tx_processed=0 - if [[ -n "$tx_hashes" ]]; then - while IFS='|' read -r tx_hash block_num tx_index; do - if [[ -n "$tx_hash" ]]; then - # Add small delay every 5 transactions to avoid rate limiting - if [[ $((tx_processed % 5)) -eq 0 && $tx_processed -gt 0 ]]; then - sleep 0.1 - fi - ((tx_processed++)) + local tx_hashes + tx_hashes=$(echo "$trace_response" | jq -r '.result | map(.transactionHash) | unique | .[]?') + tx_hashes=$(echo "$tx_hashes" | awk 'NF' | awk '!seen[$0]++') - # Fetch actual transaction data using eth_getTransactionByHash - local tx_request=$(jq -n \ - --arg tx_hash "$tx_hash" \ - '{ - "jsonrpc": "2.0", - "method": "eth_getTransactionByHash", - "params": [$tx_hash], - "id": 1 - }') + local tx_count + tx_count=$(emit_transactions_from_hashes "$rpc_url" "$tx_hashes" "$output_file") - echo "1" >> "$RPC_COUNTER_DIR/tx_fetch.count" - local tx_response=$(retry_with_backoff 5 curl -s -X POST \ - -H "Content-Type: application/json" \ - -d "$tx_request" \ - --max-time 30 \ - "$rpc_url") + echo " Found $tx_count transactions in blocks $start_block-$end_block" >&2 + echo "$tx_count" +} - local tx_data=$(echo "$tx_response" | jq -r '.result') - - if [[ -n "$tx_data" && "$tx_data" != "null" ]]; then - local tx_from=$(echo "$tx_data" | jq -r '.from') - local tx_to=$(echo "$tx_data" | jq -r '.to // empty') - local tx_value=$(echo "$tx_data" | jq -r '.value') - local tx_input=$(echo "$tx_data" | jq -r '.input') - local tx_gas_price=$(echo "$tx_data" | jq -r '.gasPrice // "0x0"') - local tx_gas_limit=$(echo "$tx_data" | jq -r '.gas // "0x0"') - local tx_max_fee_per_gas=$(echo "$tx_data" | jq -r '.maxFeePerGas // "0x0"') - local tx_max_priority_fee_per_gas=$(echo "$tx_data" | jq -r '.maxPriorityFeePerGas // "0x0"') - - # Check if transaction succeeded on-chain - local receipt_request=$(jq -n \ - --arg tx_hash "$tx_hash" \ - '{ - "jsonrpc": "2.0", - "method": "eth_getTransactionReceipt", - "params": [$tx_hash], - "id": 1 - }') - - echo "1" >> "$RPC_COUNTER_DIR/receipt_fetch.count" - local receipt_response=$(retry_with_backoff 5 curl -s -X POST \ - -H "Content-Type: application/json" \ - -d "$receipt_request" \ - --max-time 30 \ - "$rpc_url") - - local tx_status=$(echo "$receipt_response" | jq -r '.result.status // empty') - - # Only output transaction if it succeeded (status == "0x1") - if [[ "$tx_status" == "0x1" ]]; then - echo "$tx_hash|$tx_from|$tx_to|$tx_value|$tx_input|$block_num|$tx_index|$tx_gas_price|$tx_gas_limit|$tx_max_fee_per_gas|$tx_max_priority_fee_per_gas" >> "$output_file" - ((tx_count++)) - fi +# Fetch transactions using debug_traceBlockByNumber with callTracer +fetch_transactions_debug_trace_block() { + local rpc_url="$1" + local start_block="$2" + local end_block="$3" + local target_contract="$4" + local output_file="$5" + + local target_contract_lower + target_contract_lower=$(echo "$target_contract" | tr '[:upper:]' '[:lower:]') + + local tx_hashes="" + + for ((block=start_block; block<=end_block; block++)); do + local block_hex + block_hex=$(printf "0x%x" "$block") + + echo "Fetching traces for block $block using debug_traceBlockByNumber" >&2 + + local trace_request + trace_request=$(jq -n \ + --arg block_hex "$block_hex" \ + '{ + "jsonrpc": "2.0", + "method": "debug_traceBlockByNumber", + "params": [$block_hex, {"tracer":"callTracer"}], + "id": 1 + }') + + echo "1" >> "$RPC_COUNTER_DIR/debug_trace_block.count" + local trace_response + trace_response=$(retry_with_backoff 5 curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "$trace_request" \ + --max-time 60 \ + "$rpc_url") + + if [[ -z "$trace_response" ]] || echo "$trace_response" | jq -e '.error' > /dev/null 2>&1; then + local error_msg + error_msg=$(echo "$trace_response" | jq -r '.error.message // "Unknown error"') + echo "Error: debug_traceBlockByNumber failed: $error_msg" >&2 + if is_method_unsupported "$trace_response"; then + echo "debug_traceBlockByNumber unsupported on this RPC" >&2 + echo "0" + return 2 + fi + echo "0" + return 1 + fi + + local block_tx_hashes + block_tx_hashes=$(echo "$trace_response" | jq -r --arg target "$target_contract_lower" ' + def hasTarget(node): + if (node | type) == "object" then + ((node.to? // "" | ascii_downcase) == $target) or + ([node.calls[]? | hasTarget(.)] | any) + else + false + end; + .result[]? + | (.result? // .) as $r + | select(hasTarget($r)) + | (.txHash // .transactionHash // .hash // empty) + ') + + if [[ -n "$block_tx_hashes" ]]; then + tx_hashes+="$block_tx_hashes"$'\n' + fi + done + + tx_hashes=$(echo "$tx_hashes" | awk 'NF' | awk '!seen[$0]++') + local tx_count + tx_count=$(emit_transactions_from_hashes "$rpc_url" "$tx_hashes" "$output_file") + + echo " Found $tx_count transactions in blocks $start_block-$end_block" >&2 + echo "$tx_count" +} + +# Fetch transactions using debug_traceTransaction with callTracer (slow fallback) +fetch_transactions_debug_trace_tx() { + local rpc_url="$1" + local start_block="$2" + local end_block="$3" + local target_contract="$4" + local output_file="$5" + + local target_contract_lower + target_contract_lower=$(echo "$target_contract" | tr '[:upper:]' '[:lower:]') + + local tx_hashes="" + + for ((block=start_block; block<=end_block; block++)); do + local block_hex + block_hex=$(printf "0x%x" "$block") + + local block_request + block_request=$(jq -n \ + --arg block_hex "$block_hex" \ + '{ + "jsonrpc": "2.0", + "method": "eth_getBlockByNumber", + "params": [$block_hex, false], + "id": 1 + }') + + echo "1" >> "$RPC_COUNTER_DIR/block_fetch.count" + local block_response + block_response=$(retry_with_backoff 5 curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "$block_request" \ + --max-time 30 \ + "$rpc_url") + + if [[ -z "$block_response" ]] || echo "$block_response" | jq -e '.error' > /dev/null 2>&1; then + local error_msg + error_msg=$(echo "$block_response" | jq -r '.error.message // "Unknown error"') + echo "Error: eth_getBlockByNumber failed: $error_msg" >&2 + echo "0" + return 1 + fi + + local tx_list + tx_list=$(echo "$block_response" | jq -r '.result.transactions[]? // empty') + + while IFS= read -r tx_hash; do + [[ -z "$tx_hash" ]] && continue + + local trace_request + trace_request=$(jq -n \ + --arg tx_hash "$tx_hash" \ + '{ + "jsonrpc": "2.0", + "method": "debug_traceTransaction", + "params": [$tx_hash, {"tracer":"callTracer"}], + "id": 1 + }') + + echo "1" >> "$RPC_COUNTER_DIR/debug_trace_tx.count" + local trace_response + trace_response=$(retry_with_backoff 5 curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "$trace_request" \ + --max-time 60 \ + "$rpc_url") + + if [[ -z "$trace_response" ]] || echo "$trace_response" | jq -e '.error' > /dev/null 2>&1; then + local error_msg + error_msg=$(echo "$trace_response" | jq -r '.error.message // "Unknown error"') + echo "Error: debug_traceTransaction failed: $error_msg" >&2 + if is_method_unsupported "$trace_response"; then + echo "debug_traceTransaction unsupported on this RPC" >&2 + echo "0" + return 2 fi + continue fi - done <<< "$tx_hashes" - fi + + if echo "$trace_response" | jq -e --arg target "$target_contract_lower" ' + def hasTarget(node): + if (node | type) == "object" then + ((node.to? // "" | ascii_downcase) == $target) or + ([node.calls[]? | hasTarget(.)] | any) + else + false + end; + (.result? // empty) as $r + | $r != null and hasTarget($r) + ' > /dev/null; then + tx_hashes+="$tx_hash"$'\n' + fi + done <<< "$tx_list" + done + + tx_hashes=$(echo "$tx_hashes" | awk 'NF' | awk '!seen[$0]++') + local tx_count + tx_count=$(emit_transactions_from_hashes "$rpc_url" "$tx_hashes" "$output_file") echo " Found $tx_count transactions in blocks $start_block-$end_block" >&2 echo "$tx_count" @@ -670,8 +907,10 @@ main() { # Choose processing method and batch size local batch_size if [[ "$USE_TRACE_FILTER" == "true" ]]; then - echo "Starting transaction fetch using trace_filter (includes internal calls)" >&2 + TRACE_METHOD="trace_filter" + echo "Starting transaction fetch using traces (includes internal calls)" >&2 echo "Blocks: $start_block to $end_block (trace_filter batch size: $TRACE_FILTER_BATCH_SIZE)" >&2 + echo "Trace method: $TRACE_METHOD (will auto-fallback if unsupported)" >&2 batch_size=$TRACE_FILTER_BATCH_SIZE else echo "Starting transaction fetch (direct calls only)" >&2 @@ -692,7 +931,35 @@ main() { # Call appropriate processing function local tx_count if [[ "$USE_TRACE_FILTER" == "true" ]]; then - tx_count=$(fetch_transactions_trace_filter "$rpc_url" "$batch_start" "$batch_end" "$target_contract" "$batch_file") + local status=0 + while true; do + if [[ "$TRACE_METHOD" == "trace_filter" ]]; then + tx_count=$(fetch_transactions_trace_filter "$rpc_url" "$batch_start" "$batch_end" "$target_contract" "$batch_file") + status=$? + if [[ $status -eq 2 ]]; then + TRACE_METHOD="debug_trace_block" + echo "Switching to debug_traceBlockByNumber for internal call detection" >&2 + continue + fi + elif [[ "$TRACE_METHOD" == "debug_trace_block" ]]; then + tx_count=$(fetch_transactions_debug_trace_block "$rpc_url" "$batch_start" "$batch_end" "$target_contract" "$batch_file") + status=$? + if [[ $status -eq 2 ]]; then + TRACE_METHOD="debug_trace_tx" + echo "Switching to debug_traceTransaction for internal call detection" >&2 + continue + fi + else + tx_count=$(fetch_transactions_debug_trace_tx "$rpc_url" "$batch_start" "$batch_end" "$target_contract" "$batch_file") + status=$? + if [[ $status -eq 2 ]]; then + echo "No supported trace method found on this RPC; falling back to direct calls only" >&2 + TRACE_METHOD="" + tx_count=$(process_batch "$rpc_url" "$batch_start" "$batch_end" "$target_contract" "$batch_id" "$MAX_CONCURRENT") + fi + fi + break + done else tx_count=$(process_batch "$rpc_url" "$batch_start" "$batch_end" "$target_contract" "$batch_id" "$MAX_CONCURRENT") fi @@ -725,10 +992,12 @@ main() { local block_fetch_count=$(count_rpc_calls "block_fetch") local detailed_block_count=$(count_rpc_calls "detailed_block") local trace_filter_count=$(count_rpc_calls "trace_filter") + local debug_trace_block_count=$(count_rpc_calls "debug_trace_block") + local debug_trace_tx_count=$(count_rpc_calls "debug_trace_tx") local tx_fetch_count=$(count_rpc_calls "tx_fetch") local receipt_fetch_count=$(count_rpc_calls "receipt_fetch") - local total_rpc_calls=$((block_fetch_count + detailed_block_count + trace_filter_count + tx_fetch_count + receipt_fetch_count)) + local total_rpc_calls=$((block_fetch_count + detailed_block_count + trace_filter_count + debug_trace_block_count + debug_trace_tx_count + tx_fetch_count + receipt_fetch_count)) echo "" >&2 echo "=== RPC CALL STATISTICS ===" >&2 @@ -737,6 +1006,12 @@ main() { if [[ $trace_filter_count -gt 0 ]]; then echo " - trace_filter calls: $trace_filter_count" >&2 fi + if [[ $debug_trace_block_count -gt 0 ]]; then + echo " - debug_traceBlockByNumber calls: $debug_trace_block_count" >&2 + fi + if [[ $debug_trace_tx_count -gt 0 ]]; then + echo " - debug_traceTransaction calls: $debug_trace_tx_count" >&2 + fi if [[ $tx_fetch_count -gt 0 ]]; then echo " - Transaction fetches (eth_getTransactionByHash): $tx_fetch_count" >&2 fi diff --git a/src/CredibleTestWithBacktesting.sol b/src/CredibleTestWithBacktesting.sol index 93554cc..4deeeff 100644 --- a/src/CredibleTestWithBacktesting.sol +++ b/src/CredibleTestWithBacktesting.sol @@ -180,6 +180,12 @@ abstract contract CredibleTestWithBacktesting is CredibleTest, Test { inputs[16] = "--use-trace-filter"; } + string memory command = inputs[0]; + for (uint256 i = 1; i < inputs.length; i++) { + command = string.concat(command, " ", inputs[i]); + } + console.log(string.concat("FFI command: ", command)); + // Execute FFI bytes memory result = vm.ffi(inputs); string memory output = string(result); From 339d81b87819239e076a84b843fd5cc53cafc51e Mon Sep 17 00:00:00 2001 From: "Odysseas.eth" Date: Sat, 17 Jan 2026 02:12:03 +0100 Subject: [PATCH 7/7] Fix shellcheck warnings --- scripts/backtesting/transaction_fetcher.sh | 136 ++++++++++++++------- 1 file changed, 91 insertions(+), 45 deletions(-) diff --git a/scripts/backtesting/transaction_fetcher.sh b/scripts/backtesting/transaction_fetcher.sh index 594068b..5abd6d2 100755 --- a/scripts/backtesting/transaction_fetcher.sh +++ b/scripts/backtesting/transaction_fetcher.sh @@ -99,36 +99,49 @@ retry_with_backoff() { if [[ -z "$response" ]]; then attempt=$((attempt + 1)) if [[ $attempt -lt $max_retries ]]; then - local wait_time=$((2 ** attempt)) - local jitter=$((RANDOM % 1000)) - local total_wait=$((wait_time * 1000 + jitter)) - local max_wait=64000 + local wait_time + local jitter + local total_wait + local max_wait + local wait_seconds + wait_time=$((2 ** attempt)) + jitter=$((RANDOM % 1000)) + total_wait=$((wait_time * 1000 + jitter)) + max_wait=64000 if [[ $total_wait -gt $max_wait ]]; then total_wait=$max_wait fi - sleep $(awk "BEGIN {print $total_wait/1000}") + wait_seconds=$(awk "BEGIN {print $total_wait/1000}") + sleep "$wait_seconds" continue fi fi # Check for 429 error - local error_code=$(echo "$response" | jq -r '.error.code // empty' 2>/dev/null) + local error_code + error_code=$(echo "$response" | jq -r '.error.code // empty' 2>/dev/null) if [[ "$error_code" == "429" ]]; then attempt=$((attempt + 1)) if [[ $attempt -lt $max_retries ]]; then # Exponential backoff: 2^n seconds + random jitter (0-1000ms) - local wait_time=$((2 ** attempt)) - local jitter=$((RANDOM % 1000)) - local total_wait=$((wait_time * 1000 + jitter)) + local wait_time + local jitter + local total_wait + local max_wait + local wait_seconds + wait_time=$((2 ** attempt)) + jitter=$((RANDOM % 1000)) + total_wait=$((wait_time * 1000 + jitter)) # Cap at maximum backoff of 64 seconds - local max_wait=64000 + max_wait=64000 if [[ $total_wait -gt $max_wait ]]; then total_wait=$max_wait fi - echo "Rate limit hit (429), retrying after $(awk "BEGIN {print $total_wait/1000}")s (attempt $attempt/$max_retries)" >&2 - sleep $(awk "BEGIN {print $total_wait/1000}") + wait_seconds=$(awk "BEGIN {print $total_wait/1000}") + echo "Rate limit hit (429), retrying after ${wait_seconds}s (attempt $attempt/$max_retries)" >&2 + sleep "$wait_seconds" continue else echo "Max retries reached for rate-limited request" >&2 @@ -320,13 +333,16 @@ fetch_transactions_trace_filter() { local output_file="$5" # Convert block numbers to hex - local start_hex=$(printf "0x%x" "$start_block") - local end_hex=$(printf "0x%x" "$end_block") + local start_hex + local end_hex + start_hex=$(printf "0x%x" "$start_block") + end_hex=$(printf "0x%x" "$end_block") echo "Fetching traces for blocks $start_block to $end_block using trace_filter" >&2 # Prepare trace_filter request - local trace_request=$(jq -n \ + local trace_request + trace_request=$(jq -n \ --arg start_hex "$start_hex" \ --arg end_hex "$end_hex" \ --arg target "$target_contract" \ @@ -566,10 +582,12 @@ fetch_block_transactions() { local rpc_counter_dir="$5" # Convert block number to hex - local block_hex=$(printf "0x%x" "$block_number") + local block_hex + block_hex=$(printf "0x%x" "$block_number") # Prepare RPC request - local rpc_request=$(jq -n \ + local rpc_request + rpc_request=$(jq -n \ --arg method "eth_getBlockByNumber" \ --arg block_hex "$block_hex" \ '{ @@ -631,24 +649,30 @@ fetch_block_transactions() { fi # Convert block number to decimal - local block_num_decimal=$(hex_to_decimal "$block_num_hex") + local block_num_decimal + block_num_decimal=$(hex_to_decimal "$block_num_hex") # Filter transactions that interact with the target contract - local target_contract_lower=$(echo "$target_contract" | tr '[:upper:]' '[:lower:]') + local target_contract_lower + target_contract_lower=$(echo "$target_contract" | tr '[:upper:]' '[:lower:]') # Process each transaction - only check direct calls (tx.to == target) while IFS= read -r tx; do [[ -z "$tx" ]] && continue - local tx_hash=$(echo "$tx" | jq -r '.hash') - local tx_to=$(echo "$tx" | jq -r '.to // empty') + local tx_hash + local tx_to + tx_hash=$(echo "$tx" | jq -r '.hash') + tx_to=$(echo "$tx" | jq -r '.to // empty') # Check if this is a direct call to target contract if [[ -n "$tx_to" ]]; then - local tx_to_lower=$(echo "$tx_to" | tr '[:upper:]' '[:lower:]') + local tx_to_lower + tx_to_lower=$(echo "$tx_to" | tr '[:upper:]' '[:lower:]') if [[ "$tx_to_lower" == "$target_contract_lower" ]]; then # Direct call found - check if transaction succeeded on-chain - local receipt_request=$(jq -n \ + local receipt_request + receipt_request=$(jq -n \ --arg tx_hash "$tx_hash" \ '{ "jsonrpc": "2.0", @@ -658,27 +682,38 @@ fetch_block_transactions() { }') echo "1" >> "$rpc_counter_dir/receipt_fetch.count" - local receipt_response=$(retry_with_backoff 5 curl -s -X POST \ + local receipt_response + receipt_response=$(retry_with_backoff 5 curl -s -X POST \ -H "Content-Type: application/json" \ -d "$receipt_request" \ --max-time 30 \ "$rpc_url") - local tx_status=$(echo "$receipt_response" | jq -r '.result.status // empty') + local tx_status + tx_status=$(echo "$receipt_response" | jq -r '.result.status // empty') # Only output transaction if it succeeded (status == "0x1") if [[ "$tx_status" == "0x1" ]]; then - local tx_from=$(echo "$tx" | jq -r '.from') - local tx_value=$(echo "$tx" | jq -r '.value') - local tx_input=$(echo "$tx" | jq -r '.input') - local tx_index_hex=$(echo "$tx" | jq -r '.transactionIndex') - local tx_gas_price=$(echo "$tx" | jq -r '.gasPrice // "0x0"') - local tx_gas_limit=$(echo "$tx" | jq -r '.gas // "0x0"') - local tx_max_fee_per_gas=$(echo "$tx" | jq -r '.maxFeePerGas // "0x0"') - local tx_max_priority_fee_per_gas=$(echo "$tx" | jq -r '.maxPriorityFeePerGas // "0x0"') + local tx_from + local tx_value + local tx_input + local tx_index_hex + local tx_gas_price + local tx_gas_limit + local tx_max_fee_per_gas + local tx_max_priority_fee_per_gas + tx_from=$(echo "$tx" | jq -r '.from') + tx_value=$(echo "$tx" | jq -r '.value') + tx_input=$(echo "$tx" | jq -r '.input') + tx_index_hex=$(echo "$tx" | jq -r '.transactionIndex') + tx_gas_price=$(echo "$tx" | jq -r '.gasPrice // "0x0"') + tx_gas_limit=$(echo "$tx" | jq -r '.gas // "0x0"') + tx_max_fee_per_gas=$(echo "$tx" | jq -r '.maxFeePerGas // "0x0"') + tx_max_priority_fee_per_gas=$(echo "$tx" | jq -r '.maxPriorityFeePerGas // "0x0"') # Convert transaction index to decimal - local tx_index_decimal=$(hex_to_decimal "$tx_index_hex") + local tx_index_decimal + tx_index_decimal=$(hex_to_decimal "$tx_index_hex") # Output transaction in the format: hash|from|to|value|data|blockNumber|txIndex|gasPrice|gasLimit|maxFeePerGas|maxPriorityFeePerGas echo "$tx_hash|$tx_from|$tx_to|$tx_value|$tx_input|$block_num_decimal|$tx_index_decimal|$tx_gas_price|$tx_gas_limit|$tx_max_fee_per_gas|$tx_max_priority_fee_per_gas" >> "$output_file" @@ -758,7 +793,8 @@ format_transactions() { fi # Count transactions (one per line) - local tx_count=$(wc -l < "$all_transactions_file" | tr -d ' ') + local tx_count + tx_count=$(wc -l < "$all_transactions_file" | tr -d ' ') case "$output_format" in "json") @@ -976,7 +1012,8 @@ main() { done # Calculate timing - local end_time=$(date +%s) + local end_time + end_time=$(date +%s) local duration=$((end_time - START_TIME)) echo "Optimized fetch completed in ${duration}s" >&2 @@ -989,13 +1026,20 @@ main() { fi # Display RPC call statistics - local block_fetch_count=$(count_rpc_calls "block_fetch") - local detailed_block_count=$(count_rpc_calls "detailed_block") - local trace_filter_count=$(count_rpc_calls "trace_filter") - local debug_trace_block_count=$(count_rpc_calls "debug_trace_block") - local debug_trace_tx_count=$(count_rpc_calls "debug_trace_tx") - local tx_fetch_count=$(count_rpc_calls "tx_fetch") - local receipt_fetch_count=$(count_rpc_calls "receipt_fetch") + local block_fetch_count + local detailed_block_count + local trace_filter_count + local debug_trace_block_count + local debug_trace_tx_count + local tx_fetch_count + local receipt_fetch_count + block_fetch_count=$(count_rpc_calls "block_fetch") + detailed_block_count=$(count_rpc_calls "detailed_block") + trace_filter_count=$(count_rpc_calls "trace_filter") + debug_trace_block_count=$(count_rpc_calls "debug_trace_block") + debug_trace_tx_count=$(count_rpc_calls "debug_trace_tx") + tx_fetch_count=$(count_rpc_calls "tx_fetch") + receipt_fetch_count=$(count_rpc_calls "receipt_fetch") local total_rpc_calls=$((block_fetch_count + detailed_block_count + trace_filter_count + debug_trace_block_count + debug_trace_tx_count + tx_fetch_count + receipt_fetch_count)) @@ -1051,9 +1095,11 @@ main() { # Get total tx count for each block and format output for ((block = start_block; block <= end_block; block++)); do - local block_hex=$(printf "0x%x" "$block") + local block_hex + block_hex=$(printf "0x%x" "$block") echo "1" >> "$RPC_COUNTER_DIR/detailed_block.count" - local total_tx_count=$(curl -s -X POST "$rpc_url" \ + local total_tx_count + total_tx_count=$(curl -s -X POST "$rpc_url" \ -H "Content-Type: application/json" \ -d "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBlockByNumber\",\"params\":[\"$block_hex\", false],\"id\":1}" \ | jq -r '.result.transactions | length')