Add benchmark for transaction execution (#11509)

* Add benchmark for transaction execution

* Address review grumbles

* Address review grumbles and extend benches to test both blocks with both Constantinople and Istanbul rules
This commit is contained in:
David 2020-02-25 11:25:02 +01:00 committed by GitHub
parent fa6b6a19a4
commit ad56eb48b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 263 additions and 50 deletions

153
Cargo.lock generated
View File

@ -231,6 +231,12 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
[[package]]
name = "autocfg"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
[[package]]
name = "backtrace"
version = "0.3.43"
@ -306,7 +312,7 @@ version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f04a5e50dc80b3d5d35320889053637d15011aed5e66b66b37ae798c65da6f7"
dependencies = [
"autocfg",
"autocfg 0.1.7",
"byteorder",
"serde",
]
@ -761,9 +767,9 @@ dependencies = [
[[package]]
name = "criterion"
version = "0.3.0"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "938703e165481c8d612ea3479ac8342e5615185db37765162e762ec3523e2fc6"
checksum = "1fc755679c12bda8e5523a71e4d654b6bf2e14bd838dfc48cde6559a05caf7d1"
dependencies = [
"atty",
"cast",
@ -773,10 +779,10 @@ dependencies = [
"itertools",
"lazy_static",
"num-traits 0.2.6",
"rand_core 0.5.1",
"rand_os 0.2.2",
"rand_xoshiro",
"oorandom",
"plotters",
"rayon",
"regex",
"serde",
"serde_derive",
"serde_json",
@ -786,9 +792,9 @@ dependencies = [
[[package]]
name = "criterion-plot"
version = "0.4.0"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eccdc6ce8bbe352ca89025bee672aa6d24f4eb8c53e3a8b5d1bc58011da072a2"
checksum = "a01e15e0ea58e8234f96146b1f91fa9d0e4dd7a38da93ff7a75d42c0b9d3a545"
dependencies = [
"cast",
"itertools",
@ -800,18 +806,19 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05e44b8cf3e1a625844d1750e1f7820da46044ff6d28f4d43e455ba3e5bb2c13"
dependencies = [
"crossbeam-epoch",
"crossbeam-epoch 0.7.2",
"crossbeam-utils 0.6.6",
]
[[package]]
name = "crossbeam-deque"
version = "0.7.1"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b18cd2e169ad86297e6bc0ad9aa679aee9daa4f19e8163860faf7c164e4f5a71"
checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils 0.6.6",
"crossbeam-epoch 0.8.2",
"crossbeam-utils 0.7.2",
"maybe-uninit",
]
[[package]]
@ -828,6 +835,21 @@ dependencies = [
"scopeguard 1.0.0",
]
[[package]]
name = "crossbeam-epoch"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
dependencies = [
"autocfg 1.0.0",
"cfg-if",
"crossbeam-utils 0.7.2",
"lazy_static",
"maybe-uninit",
"memoffset",
"scopeguard 1.0.0",
]
[[package]]
name = "crossbeam-queue"
version = "0.1.2"
@ -837,6 +859,16 @@ dependencies = [
"crossbeam-utils 0.6.6",
]
[[package]]
name = "crossbeam-queue"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db"
dependencies = [
"cfg-if",
"crossbeam-utils 0.7.2",
]
[[package]]
name = "crossbeam-utils"
version = "0.5.0"
@ -853,6 +885,17 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg 1.0.0",
"cfg-if",
"lazy_static",
]
[[package]]
name = "crunchy"
version = "0.1.6"
@ -1805,12 +1848,15 @@ dependencies = [
"account-db",
"account-state",
"common-types",
"criterion",
"env_logger 0.5.13",
"ethcore",
"ethcore-db",
"ethereum-types",
"evm",
"hash-db",
"hex-literal",
"journaldb",
"keccak-hash",
"keccak-hasher 0.1.1",
"kvdb",
@ -1821,6 +1867,8 @@ dependencies = [
"patricia-trie-ethereum",
"pod",
"spec",
"state-db",
"tempdir",
"trace",
"trie-db",
"trie-vm-factories",
@ -2090,7 +2138,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead"
dependencies = [
"ahash",
"autocfg",
"autocfg 0.1.7",
]
[[package]]
@ -2304,7 +2352,7 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712d7b3ea5827fcb9d4fda14bf4da5f136f0db2ae9c8f4bd4e2d1c6fde4e6db2"
dependencies = [
"autocfg",
"autocfg 0.1.7",
]
[[package]]
@ -2799,6 +2847,12 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
[[package]]
name = "maybe-uninit"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]]
name = "memchr"
version = "2.2.1"
@ -3140,6 +3194,12 @@ dependencies = [
"winapi-build",
]
[[package]]
name = "oorandom"
version = "11.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebcec7c9c2a95cacc7cd0ecb89d8a8454eca13906f6deb55258ffff0adeb9405"
[[package]]
name = "opaque-debug"
version = "0.2.2"
@ -3771,6 +3831,18 @@ dependencies = [
"crunchy 0.1.6",
]
[[package]]
name = "plotters"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e3bb8da247d27ae212529352020f3e5ee16e83c0c258061d27b08ab92675eeb"
dependencies = [
"js-sys",
"num-traits 0.2.6",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "pod"
version = "0.1.0"
@ -4002,14 +4074,14 @@ version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
dependencies = [
"autocfg",
"autocfg 0.1.7",
"libc",
"rand_chacha 0.1.1",
"rand_core 0.4.2",
"rand_hc 0.1.0",
"rand_isaac",
"rand_jitter",
"rand_os 0.1.3",
"rand_os",
"rand_pcg",
"rand_xorshift 0.1.1",
"winapi 0.3.8",
@ -4034,7 +4106,7 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
dependencies = [
"autocfg",
"autocfg 0.1.7",
"rand_core 0.3.1",
]
@ -4133,23 +4205,13 @@ dependencies = [
"winapi 0.3.8",
]
[[package]]
name = "rand_os"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a788ae3edb696cfcba1c19bfd388cc4b8c21f8a408432b199c072825084da58a"
dependencies = [
"getrandom",
"rand_core 0.5.1",
]
[[package]]
name = "rand_pcg"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
dependencies = [
"autocfg",
"autocfg 0.1.7",
"rand_core 0.4.2",
]
@ -4171,35 +4233,26 @@ dependencies = [
"rand_core 0.5.1",
]
[[package]]
name = "rand_xoshiro"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e18c91676f670f6f0312764c759405f13afb98d5d73819840cf72a518487bff"
dependencies = [
"rand_core 0.5.1",
]
[[package]]
name = "rayon"
version = "1.1.0"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4b0186e22767d5b9738a05eab7c6ac90b15db17e5b5f9bd87976dd7d89a10a4"
checksum = "db6ce3297f9c85e16621bb8cca38a06779ffc31bb8184e1be4bed2be4678a098"
dependencies = [
"crossbeam-deque 0.6.3",
"crossbeam-deque 0.7.3",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.5.0"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebbe0df8435ac0c397d467b6cad6d25543d06e8a019ef3f6af3c384597515bd2"
checksum = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9"
dependencies = [
"crossbeam-deque 0.6.3",
"crossbeam-queue",
"crossbeam-utils 0.6.6",
"crossbeam-deque 0.7.3",
"crossbeam-queue 0.2.1",
"crossbeam-utils 0.7.2",
"lazy_static",
"num_cpus",
]
@ -5255,8 +5308,8 @@ version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bd2c6a3885302581f4401c82af70d792bb9df1700e7437b0aeb4ada94d5388c"
dependencies = [
"crossbeam-deque 0.7.1",
"crossbeam-queue",
"crossbeam-deque 0.7.3",
"crossbeam-queue 0.1.2",
"crossbeam-utils 0.6.6",
"futures",
"lazy_static",
@ -5693,9 +5746,9 @@ dependencies = [
[[package]]
name = "walkdir"
version = "2.2.7"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d9d7ed3431229a144296213105a390676cc49c9b6a72bd19f3176c98e129fa1"
checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
dependencies = [
"same-file",
"winapi 0.3.8",

View File

@ -6,6 +6,10 @@ authors = ["Parity Technologies <admin@parity.io>"]
edition = "2018"
license = "GPL-3.0"
[[bench]]
name = "execution"
harness = false
[dependencies]
account-db = { path = "../account-db" }
account-state = { path = "../account-state" }
@ -32,3 +36,9 @@ hex-literal = "0.2.1"
spec = { path = "../spec" }
trie-db = "0.20.0"
ethtrie = { package = "patricia-trie-ethereum", path = "../../util/patricia-trie-ethereum" }
# Benchmarks
criterion = "0.3.1"
ethcore-db = { path = "../db" }
journaldb = { path = "../../util/journaldb" }
state-db = { path = "../state-db" }
tempdir = "0.3.7"

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1,150 @@
// Copyright 2015-2020 Parity Technologies (UK) Ltd.
// This file is part of Parity Ethereum.
// Parity Ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Parity Ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Parity Ethereum. If not, see <http://www.gnu.org/licenses/>.
//! Benchmark transaction execution of two blocks from mainnet, both of average
//! size (~35kb as RLP), one with 229 transactions (#8481475, Constantinople era)
//! and the other with 139 transactions (9532543, Istanbul era). Note that the
//! benchmark here is almost completely CPU-bound and does not involve IO at all,
//! so be careful not to draw too many conclusions from the results.
use std::time::{Duration, Instant};
use criterion::{Criterion, criterion_group, criterion_main};
use account_state::{CleanupMode, State};
use common_types::{
header::Header,
transaction::SignedTransaction,
verification::Unverified
};
use ethcore::test_helpers::new_temp_db;
use ethcore_db as db;
use ethereum_types::U256;
use executive_state::ExecutiveState;
use spec::{new_constantinople_test_machine, new_istanbul_test_machine};
use state_db::StateDB;
use tempdir::TempDir;
fn build_state() -> State<StateDB> {
let db_path = TempDir::new("execution-bench").unwrap();
let db = new_temp_db(&db_path.path());
let journal_db = journaldb::new(db.key_value().clone(), journaldb::Algorithm::OverlayRecent, db::COL_STATE);
let state_db = StateDB::new(journal_db, 25 * 1024 * 1024);
State::new(state_db, U256::zero(), Default::default())
}
fn setup_state_for_block(state: &mut State<StateDB>, block: Unverified) -> Vec<SignedTransaction> {
block.transactions
.into_iter()
.map(|tx| tx.verify_unordered().expect("tx is from known-good block"))
.inspect(|tx| {
// Ensure we have enough cash to execute the transaction
let gas_cost = tx.gas * tx.gas_price;
state.add_balance(&tx.sender(), &(tx.value + gas_cost), CleanupMode::ForceCreate).unwrap();
// Fix up the nonce such that the state has the expected nonce
if state.nonce(&tx.sender()).unwrap() == U256::zero() {
for _ in 0..tx.nonce.as_usize() {
state.inc_nonce(&tx.sender()).unwrap();
}
}
})
.collect::<Vec<_>>()
}
fn build_env_info(header: &Header) -> vm::EnvInfo {
vm::EnvInfo {
number: header.number(),
author: *header.author(),
timestamp: header.timestamp(),
difficulty: *header.difficulty(),
gas_limit: *header.gas_limit() * 10,
last_hashes: std::sync::Arc::new(vec![]),
gas_used: *header.gas_used(),
}
}
macro_rules! bench_tx_apply {
($b: expr, $state: expr, $env_info: expr, $machine: expr, $signed_txs: expr, tracing => $tracing: expr ) => {
$b.iter_custom(|iters| {
let mut dur = Duration::new(0, 0);
for _ in 0..iters {
$state.checkpoint();
let start = Instant::now();
for tx in &$signed_txs {
let outcome = $state.apply(&$env_info, &$machine, tx, $tracing);
assert!(outcome.is_ok())
}
dur += start.elapsed();
$state.revert_to_checkpoint();
}
dur
})
}
}
fn execute_8481475(c: &mut Criterion) {
// Block from the Constantinople era; 202 transactions, 32k RLP
let constantinople_block = Unverified::from_rlp(include_bytes!("./8481475.rlp").to_vec()).unwrap();
let mut state = build_state();
let env_info = build_env_info(&constantinople_block.header);
let signed_txs = setup_state_for_block(&mut state, constantinople_block);
let machine = new_constantinople_test_machine();
c.bench_function("Block 8481475, apply txs (Costantinople, tracing)", |b| {
bench_tx_apply!(b, state, env_info, machine, signed_txs, tracing => true);
});
c.bench_function("Block 8481475, apply txs (Costantinople, no tracing)", |b| {
bench_tx_apply!(b, state, env_info, machine, signed_txs, tracing => false);
});
let machine = new_istanbul_test_machine();
c.bench_function("Block 8481475, apply txs (Istanbul, tracing)", |b| {
bench_tx_apply!(b, state, env_info, machine, signed_txs, tracing => true);
});
c.bench_function("Block 8481475, apply txs (Istanbul, no tracing)", |b| {
bench_tx_apply!(b, state, env_info, machine, signed_txs, tracing => false);
});
}
fn execute_9532543(c: &mut Criterion) {
// Block from the Istanbul era; 139 transactions, 38k RLP
let istanbul_block = Unverified::from_rlp(include_bytes!("./9532543.rlp").to_vec()).unwrap();
let mut state = build_state();
let env_info = build_env_info(&istanbul_block.header);
let signed_txs = setup_state_for_block(&mut state, istanbul_block);
let machine = new_constantinople_test_machine();
c.bench_function("Block 9532543, apply txs (Constantinople, tracing)", |b| {
bench_tx_apply!(b, state, env_info, machine, signed_txs, tracing => true);
});
c.bench_function("Block 9532543, apply txs (Constantinople, no tracing)", |b| {
bench_tx_apply!(b, state, env_info, machine, signed_txs, tracing => false);
});
let machine = new_istanbul_test_machine();
c.bench_function("Block 9532543, apply txs (Istanbul, tracing)", |b| {
bench_tx_apply!(b, state, env_info, machine, signed_txs, tracing => true);
});
c.bench_function("Block 9532543, apply txs (Istanbul, no tracing)", |b| {
bench_tx_apply!(b, state, env_info, machine, signed_txs, tracing => false);
});
}
criterion_group!(benches, execute_8481475, execute_9532543);
criterion_main!(benches);