From 64d368282c910afaeeb41bf66998425ec7fa96f1 Mon Sep 17 00:00:00 2001 From: Karol Kokoszka Date: Sun, 21 Jan 2024 11:04:30 +0100 Subject: [PATCH 1/4] Improve TPS calculation (ignore non-call txs, ignore last block) --- src/evm/runner.rs | 40 ++++++++++++++++++++++--------- src/stats.rs | 60 +++++++++++++++++++++++++++++++++++----------- src/wasm/runner.rs | 41 ++++++++++++++++--------------- 3 files changed, 97 insertions(+), 44 deletions(-) diff --git a/src/evm/runner.rs b/src/evm/runner.rs index ede7acbc..01be0208 100644 --- a/src/evm/runner.rs +++ b/src/evm/runner.rs @@ -3,7 +3,9 @@ use std::collections::HashSet; use super::xts::{ api::{ self, + ethereum::calls::types::Transact, ethereum::events::Executed, + runtime_types::ethereum::transaction::{TransactionAction, TransactionV2}, runtime_types::evm_core::error::{ExitReason, ExitSucceed}, }, MoonbeamApi, @@ -25,7 +27,7 @@ pub struct MoonbeamRunner { pub api: MoonbeamApi, signer: SecretKey, address: Address, - calls: Vec<(String, Vec)>, + calls: Vec<(String, Vec)>, } impl MoonbeamRunner { @@ -93,7 +95,7 @@ impl MoonbeamRunner { .estimate_gas(self.address, Some(contract), &data) .await .note("Error estimating gas")?; - calls.push(Call { + calls.push(RunnerCall { name: name.to_string(), contract, data, @@ -210,17 +212,33 @@ impl MoonbeamRunner { client: OnlineClient, block_hash: sp_core::H256, ) -> color_eyre::Result> { - let events = client.events().at(block_hash).await?; + let block = client.blocks().at(block_hash).await?; let mut tx_hashes = Vec::new(); - for event in events.iter() { - let event = event?; - if let Some(Executed { - transaction_hash, .. - }) = event.as_event::()? - { - tx_hashes.push(transaction_hash); + let extrinsics_details = block + .extrinsics() + .await? + .iter() + .collect::, _>>()?; + + for extrinsic_detail in extrinsics_details { + if let Some(Transact { transaction }) = extrinsic_detail.as_extrinsic::()? { + if let TransactionV2::Legacy(tx) = transaction { + if let TransactionAction::Call(_) = tx.action { + let events = extrinsic_detail.events().await?; + for event in events.iter() { + let event = event?; + if let Some(Executed { + transaction_hash, .. + }) = event.as_event::()? + { + tx_hashes.push(transaction_hash); + } + } + } + } } } + Ok(tx_hashes) } @@ -286,7 +304,7 @@ impl MoonbeamRunner { } } -struct Call { +struct RunnerCall { name: String, contract: Address, data: Vec, diff --git a/src/stats.rs b/src/stats.rs index 893ed5f1..fde85429 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex}; pub struct BlockInfo { pub stats: blockstats::BlockStats, // list of hashes to look for - pub hashes: Vec, + pub contract_call_hashes: Vec, } /// Subscribes to block stats. Completes once *all* hashes in `remaining_hashes` have been received. @@ -44,33 +44,65 @@ where for xt in &hashes { remaining_hashes.remove(xt); } - Ok(BlockInfo { hashes, stats }) + Ok(BlockInfo { + contract_call_hashes: hashes, + stats, + }) } }) } -/// Print the block info stats to the console +/// This function prints statistics to the standard output. + +/// The TPS calculation is based on the following assumptions about smart-bench: +/// - smart-bench instantiates smart contracts on the chain and waits for the completion of these transactions. +/// - Starting from some future block (after creation), smart-bench uploads transactions related to contract calls to the node. +/// - Sending contract call transactions to the node is continuous once started and is not mixed with any other type of transactions. +/// - Smart-bench finishes benchmarking at the block that contains the last contract call from the set. + +/// TPS calculation is exclusively concerned with contract calls, disregarding any system or contract-creating transactions. + +/// TPS calculation excludes the last block of the benchmark, as its full utilization is not guaranteed. In other words, only blocks in the middle will consist entirely of contract calls. pub async fn print_block_info( block_info: impl TryStream, ) -> color_eyre::Result<()> { - let mut total_extrinsics = 0u64; - let mut total_blocks = 0u64; + let mut call_extrinsics_per_block: Vec = Vec::new(); + let mut call_block_expected = false; println!(); block_info .try_for_each(|block| { println!("{}", block.stats); - total_extrinsics += block.stats.num_extrinsics; - total_blocks += 1; + let contract_calls_count = block.contract_call_hashes.len() as u64; + // Skip blocks at the beggining until we see first call related transaction + // Once first call is seen, we expect all further blocks to contain calls until all calls are covered + if !call_block_expected && contract_calls_count > 0 { + call_block_expected = true; + } + if call_block_expected { + call_extrinsics_per_block.push(contract_calls_count); + } future::ready(Ok(())) }) .await?; + + // Skip last block as its not stressed to its full cabailities since there is very low chance of hitting + // that exact amount of transactions (it will contain as much transactions as there are left to execute) + let call_extrinsics_per_block = + &call_extrinsics_per_block[0..call_extrinsics_per_block.len() - 1]; + + let tps_blocks = call_extrinsics_per_block.len(); + let tps_total_extrinsics = call_extrinsics_per_block.iter().sum::(); println!("\nSummary:"); - println!("Total Blocks: {total_blocks}"); - println!("Total Extrinsics: {total_extrinsics}"); - println!("TPS - Transaction execution time per second, assuming a 0.5-second execution time per block"); - println!( - "TPS: {}", - total_extrinsics as f64 / (total_blocks as f64 * 0.5) - ); + println!("Total Blocks: {tps_blocks}"); + println!("Total Extrinsics: {tps_total_extrinsics}"); + if tps_blocks > 0 { + println!("TPS - Transaction execution time per second, assuming a 0.5-second execution time per block"); + println!( + "TPS: {}", + tps_total_extrinsics as f64 / (tps_blocks as f64 * 0.5) + ); + } else { + println!("TPS - Error - not enough data to calculate TPS, consider increasing --call-count value") + } Ok(()) } diff --git a/src/wasm/runner.rs b/src/wasm/runner.rs index 37635954..b5b5c748 100644 --- a/src/wasm/runner.rs +++ b/src/wasm/runner.rs @@ -7,13 +7,17 @@ use sp_runtime::traits::{BlakeTwo256, Hash as _}; use std::time::{SystemTime, UNIX_EPOCH}; use subxt::{backend::rpc::RpcClient, OnlineClient, PolkadotConfig as DefaultConfig}; +use xts::api::{ + contracts::calls::types::Call, contracts::events::Instantiated, system::events::ExtrinsicFailed, +}; + pub const DEFAULT_STORAGE_DEPOSIT_LIMIT: Option = None; pub struct BenchRunner { url: String, api: ContractsApi, signer: Signer, - calls: Vec<(String, Vec)>, + calls: Vec<(String, Vec)>, } impl BenchRunner { @@ -68,7 +72,7 @@ impl BenchRunner { .iter() .map(|contract| { let message = create_message(); - Call { + RunnerCall { contract_account: contract.clone(), call_data: message, } @@ -134,17 +138,12 @@ impl BenchRunner { let events = block.events().await?; for event in events.iter() { let event = event?; - if let Some(instantiated) = - event.as_event::()? - { + if let Some(instantiated) = event.as_event::()? { accounts.push(instantiated.contract); if accounts.len() == count as usize { return Ok(accounts); } - } else if event - .as_event::()? - .is_some() - { + } else if event.as_event::()?.is_some() { let metadata = self.api.client.metadata(); let dispatch_error = subxt::error::DispatchError::decode_from(event.field_bytes(), metadata); @@ -166,17 +165,21 @@ impl BenchRunner { client: OnlineClient, block_hash: sp_core::H256, ) -> color_eyre::Result> { - let block = client.blocks().at(block_hash).await; - let hashes = block - .unwrap_or_else(|_| panic!("block {} not found", block_hash)) + let block = client.blocks().at(block_hash).await?; + let mut tx_hashes = Vec::new(); + let extrinsics_details = block .extrinsics() - .await - .unwrap_or_else(|_| panic!("extrinsics at block {} not found", block_hash)) + .await? .iter() - .map(|e| e.unwrap_or_else(|_| panic!("extrinsic error at block {}", block_hash))) - .map(|e| BlakeTwo256::hash_of(&e.bytes())) - .collect(); - Ok(hashes) + .collect::, _>>()?; + + for extrinsic_detail in extrinsics_details { + if let Some(Call { .. }) = extrinsic_detail.as_extrinsic::()? { + tx_hashes.push(BlakeTwo256::hash_of(&extrinsic_detail.bytes())); + } + } + + Ok(tx_hashes) } /// Call each contract instance `call_count` times. Wait for all txs to be included in a block @@ -273,7 +276,7 @@ where } #[derive(Clone)] -pub struct Call { +pub struct RunnerCall { contract_account: AccountId, call_data: EncodedMessage, } From fa335a3bc20a6483a589c9e12ef5d362da889550 Mon Sep 17 00:00:00 2001 From: Karol Kokoszka Date: Sun, 21 Jan 2024 11:48:53 +0100 Subject: [PATCH 2/4] Adapt integration tests to new TPS calculation requirements --- src/integration_tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/integration_tests.rs b/src/integration_tests.rs index 2d304919..12854dbc 100644 --- a/src/integration_tests.rs +++ b/src/integration_tests.rs @@ -164,7 +164,7 @@ async fn test_ink_contract_success() { .arg("ink-wasm") .arg("flipper") .args(["--instance-count", "1"]) - .args(["--call-count", "1"]) + .args(["--call-count", "10"]) .args(["--url", "ws://localhost:9944"]) .timeout(std::time::Duration::from_secs(5)) .output() @@ -209,7 +209,7 @@ async fn test_solidity_wasm_contract_success() { .arg("sol-wasm") .arg("flipper") .args(["--instance-count", "1"]) - .args(["--call-count", "1"]) + .args(["--call-count", "10"]) .args(["--url", "ws://localhost:9944"]) .timeout(std::time::Duration::from_secs(5)) .output() @@ -255,7 +255,7 @@ async fn test_solidity_evm_contract_success() { .arg("evm") .arg("flipper") .args(["--instance-count", "1"]) - .args(["--call-count", "1"]) + .args(["--call-count", "10"]) .args(["--url", "ws://localhost:9944"]) .timeout(std::time::Duration::from_secs(5)) .output() From 60d72434d538b8a5b8a2f96494746c3fd0311f36 Mon Sep 17 00:00:00 2001 From: Sebastian Miasojed Date: Tue, 23 Apr 2024 22:31:39 +0200 Subject: [PATCH 3/4] Add more contracts to benchmark --- .github/workflows/benchmark.yml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a81b3e3e..5e4cc947 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -11,7 +11,7 @@ env: MOONBEAM_BIN: moonbeam_release/*/target/release/moonbeam MOONBEAM_VERSION: version BENCHMARK_DIR: stats - TEST_PARAMS: --instance-count 1 --call-count 1000 + TEST_PARAMS: --instance-count 1 --call-count 2000 BENCHMARK_URI: https://raw.githubusercontent.com/paritytech/smart-bench/gh-pages jobs: @@ -70,7 +70,7 @@ jobs: strategy: matrix: type: [ink-wasm, sol-wasm, evm] - contract: [erc20] + contract: [erc20, trianglenumber, storageread, storagewrite] env: BENCHMARK_FILE: benchmark-${{ matrix.type }}-${{ matrix.contract }}.csv needs: build_dev_moonbeam @@ -217,13 +217,23 @@ jobs: - name: Merge CSV run: | - curl -L -o ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} ${{ env.BENCHMARK_URI }}/${{ env.BENCHMARK_FILE }} - cat ${{ env.BENCHMARK_DIR }}/*/*.csv >> ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} + for file in ${{ env.BENCHMARK_DIR }}/*/*.csv; do + # Extract contract name + contract_name=$(basename "$file" | sed 's/^.*_\(.*\)\.csv$/\1/') + benchmark_file=${contract_name}_bench.csv + if [ ! -f ${{ env.BENCHMARK_DIR }}/${benchmark_file} ]; then + curl -L -o ${{ env.BENCHMARK_DIR }}/${benchmark_file} ${{ env.BENCHMARK_URI }}/${benchmark_file} + fi + cat $file >> ${{ env.BENCHMARK_DIR }}/${benchmark_file} + done - name: Generate graph run: | cd stats - ./get_graph.sh --panel-id=2 --csv-data=../${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} --output=../${{ env.BENCHMARK_DIR }}/tps.png + for file in ${{ env.BENCHMARK_DIR }}/*.csv; do + contract_name=$(basename "$file" | sed 's/^.*_\(.*\)\.csv$/\1/') + ./get_graph.sh --panel-id=2 --csv-data=../${file} --output=../${{ env.BENCHMARK_DIR }}/${contract_name}_stps.png + done - name: Commit benchmark stats run: | @@ -234,12 +244,12 @@ jobs: git fetch origin gh-pages # saving stats mkdir /tmp/stats - mv ${{ env.BENCHMARK_DIR }}/${{ env.BENCHMARK_FILE }} /tmp/stats - mv ${{ env.BENCHMARK_DIR }}/tps.png /tmp/stats + mv ${{ env.BENCHMARK_DIR }}/*.csv /tmp/stats + mv ${{ env.BENCHMARK_DIR }}/*.png /tmp/stats git checkout gh-pages mv /tmp/stats/* . # Upload files - git add ${{ env.BENCHMARK_FILE }} tps.png --force + git add *.csv *.png --force git status git commit -m "Updated stats in ${CURRENT_DATE} and pushed to gh-pages" git push origin gh-pages --force From 58b65baf62a151aa7de260a7587aa3c7e2ff6583 Mon Sep 17 00:00:00 2001 From: Sebastian Miasojed Date: Fri, 26 Apr 2024 17:11:56 +0200 Subject: [PATCH 4/4] Fix benchmark issue --- .github/workflows/benchmark.yml | 23 +++++++++++------------ README.md | 17 +++++++++++++++++ src/stats.rs | 2 +- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5e4cc947..703d42c1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,4 +1,4 @@ -name: Schedule based benchmark of pallet-contracts and pallet-evm +name: Benchmark of pallet-contracts and pallet-evm on: schedule: @@ -70,9 +70,9 @@ jobs: strategy: matrix: type: [ink-wasm, sol-wasm, evm] - contract: [erc20, trianglenumber, storageread, storagewrite] + contract: [erc20, flipper, triangle-number, storage-read, storage-write] env: - BENCHMARK_FILE: benchmark-${{ matrix.type }}-${{ matrix.contract }}.csv + BENCHMARK_FILE: benchmark_${{ matrix.type }}_${{ matrix.contract }}.csv needs: build_dev_moonbeam runs-on: ubuntu-latest steps: @@ -204,8 +204,6 @@ jobs: collect: runs-on: ubuntu-latest needs: [smart_contract_benchmark] - env: - BENCHMARK_FILE: benchmark-results.csv steps: - name: Checkout uses: actions/checkout@v4 @@ -220,19 +218,20 @@ jobs: for file in ${{ env.BENCHMARK_DIR }}/*/*.csv; do # Extract contract name contract_name=$(basename "$file" | sed 's/^.*_\(.*\)\.csv$/\1/') - benchmark_file=${contract_name}_bench.csv + benchmark_file=bench_${contract_name}.csv if [ ! -f ${{ env.BENCHMARK_DIR }}/${benchmark_file} ]; then - curl -L -o ${{ env.BENCHMARK_DIR }}/${benchmark_file} ${{ env.BENCHMARK_URI }}/${benchmark_file} + curl -L -o ${{ env.BENCHMARK_DIR }}/${benchmark_file} ${{ env.BENCHMARK_URI }}/${benchmark_file} || exit 1 fi cat $file >> ${{ env.BENCHMARK_DIR }}/${benchmark_file} done - name: Generate graph run: | - cd stats - for file in ${{ env.BENCHMARK_DIR }}/*.csv; do + for file in ${{ env.BENCHMARK_DIR }}/bench_*.csv; do contract_name=$(basename "$file" | sed 's/^.*_\(.*\)\.csv$/\1/') - ./get_graph.sh --panel-id=2 --csv-data=../${file} --output=../${{ env.BENCHMARK_DIR }}/${contract_name}_stps.png + pushd stats + ./get_graph.sh --panel-id=2 --csv-data=../${file} --output=../${{ env.BENCHMARK_DIR }}/stps_${contract_name}.png + popd done - name: Commit benchmark stats @@ -244,8 +243,8 @@ jobs: git fetch origin gh-pages # saving stats mkdir /tmp/stats - mv ${{ env.BENCHMARK_DIR }}/*.csv /tmp/stats - mv ${{ env.BENCHMARK_DIR }}/*.png /tmp/stats + mv ${{ env.BENCHMARK_DIR }}/bench_*.csv /tmp/stats + mv ${{ env.BENCHMARK_DIR }}/stps_*.png /tmp/stats git checkout gh-pages mv /tmp/stats/* . # Upload files diff --git a/README.md b/README.md index 036c5bd5..abb75b26 100644 --- a/README.md +++ b/README.md @@ -117,3 +117,20 @@ Before running tests, smart-bench needs to be build using `cargo build` command. Integration tests requires two types of nodes to be installed and available on `PATH`. - [`moonbeam`](https://github.com/PureStake/moonbeam/) with enabled [`dev RPC`](https://github.com/paritytech/substrate-contracts-node/blob/539cf0271090f406cb3337e4d97680a6a63bcd2f/node/src/rpc.rs#L60) for Solidity/EVM contracts - [`substrate-contracts-node`](https://github.com/paritytech/substrate-contracts-node/) for Ink! and Solang (Solidity/Wasm) contracts + +### Benchmarks + +## Erc20 +![Erc20](https://github.com/paritytech/smart-bench/blob/gh-pages/stps_erc20.png?raw=true) + +## Flipper +![Flipper](https://github.com/paritytech/smart-bench/blob/gh-pages/stps_flipper.png?raw=true) + +## Storage Read +![Storage Read](https://github.com/paritytech/smart-bench/blob/gh-pages/stps_storage-read.png?raw=true) + +## Storage Write +![Storage Write](https://github.com/paritytech/smart-bench/blob/gh-pages/stps_storage-write.png?raw=true) + +## Triangle Number +![Triangle Number](https://github.com/paritytech/smart-bench/blob/gh-pages/stps_triangle-number.png?raw=true) diff --git a/src/stats.rs b/src/stats.rs index 6dc31820..b2f3c791 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -119,7 +119,7 @@ pub async fn print_block_info( if tps_blocks > 0 { println!("sTPS - Standard Transaction Per Second"); println!( - "sTPS: {}", + "sTPS: {:.2}", tps_total_extrinsics as f64 / (tps_blocks as f64 * diff) ); } else {