Convert std::test benchmarks to use Criterion (#10999)

* Convert std::test benches to Criterion

* Required feature for ethash benches

* gitlab-ci: Check all bench targets

* check ethash benches

* evmbin/benches/mod.rs: Update bench sample command

Co-Authored-By: Andronik Ordian <write@reusable.software>

* Upgrade to criterion 0.3

.gitlab-ci.yml

@@ -96,11 +96,14 @@ cargo-check 2 3:
     - time cargo check --target $CARGO_TARGET --locked --manifest-path util/io/Cargo.toml --features "mio" --verbose --color=always
     - sccache -s
 
-cargo-check-ethcore-benches:
+cargo-check-benches:
   stage:                           test
   <<:                              *docker-cache-status
   script:
-    - time cargo check -p ethcore --benches --target $CARGO_TARGET --locked --verbose --color=always
+    - time (
+        cargo check --all --benches --exclude ethash --target $CARGO_TARGET --locked --verbose --color=always;
+        (cd ethash; time cargo check --benches --features bench --target $CARGO_TARGET --locked --verbose --color=always)
+      )
     - sccache -s
 
 cargo-audit:

ethash/Cargo.toml

@@ -17,7 +17,7 @@ primal = "0.2.3"
 static_assertions = "0.3.3"
 
 [dev-dependencies]
-criterion = "0.2"
+criterion = "0.3"
 rustc-hex = "1.0"
 serde_json = "1.0"
 tempdir = "0.3"

ethcore/Cargo.toml

@@ -74,7 +74,7 @@ vm = { path = "vm" }
 account-db = { path = "account-db" }
 blooms-db = { path = "../util/blooms-db" }
 ethcore-builtin = { path = "./builtin" }
-criterion = "0.2"
+criterion = "0.3"
 engine = { path = "./engine", features = ["test-helpers"] }
 env_logger = "0.5"
 ethash = { path = "../ethash" }

ethcore/evm/Cargo.toml

@@ -18,7 +18,7 @@ memory-cache = { path = "../../util/memory-cache" }
 
 [dev-dependencies]
 rustc-hex = "1.0"
-criterion = "0.2"
+criterion = "0.3"
 hex-literal = "0.2.0"
 
 [features]

evmbin/Cargo.toml

@@ -29,7 +29,12 @@ trace = { path = "../ethcore/trace" }
 vm = { path = "../ethcore/vm" }
 
 [dev-dependencies]
+criterion = "0.3"
 tempdir = "0.3"
 
 [features]
 evm-debug = ["ethcore/evm-debug-tests"]
+
+[[bench]]
+name = "mod"
+harness = false

evmbin/benches/mod.rs

@@ -17,12 +17,11 @@
 //! benchmarking for EVM
 //! should be started with:
 //! ```bash
-//! rustup run nightly cargo bench
+//! cargo bench
 //! ```
 
-#![feature(test)]
+#[macro_use]
-
+extern crate criterion;
-extern crate test;
 extern crate ethcore;
 extern crate evm;
 extern crate ethereum_types;
@@ -30,7 +29,7 @@ extern crate rustc_hex;
 extern crate vm;
 
 use std::sync::Arc;
-use self::test::{Bencher, black_box};
+use criterion::{Criterion, black_box};
 
 use ethereum_types::U256;
 use evm::Factory;
@@ -38,54 +37,63 @@ use rustc_hex::FromHex;
 use vm::tests::FakeExt;
 use vm::{ActionParams, Ext};
 
-#[bench]
+criterion_group!(
-fn simple_loop_usize(b: &mut Bencher) {
+	evmbin,
-	simple_loop(U256::from(::std::usize::MAX), b)
+	bench_simple_loop_usize,
+	bench_simple_loop_u256,
+	bench_rng_usize,
+	bench_rng_u256
+);
+criterion_main!(evmbin);
+
+fn bench_simple_loop_usize(c: &mut Criterion) {
+	simple_loop(U256::from(::std::usize::MAX), c, "simple_loop_usize")
 }
 
-#[bench]
+fn bench_simple_loop_u256(c: &mut Criterion) {
-fn simple_loop_u256(b: &mut Bencher) {
+	simple_loop(!U256::zero(), c, "simple_loop_u256")
-	simple_loop(!U256::zero(), b)
 }
 
-fn simple_loop(gas: U256, b: &mut Bencher) {
+fn simple_loop(gas: U256, c: &mut Criterion, bench_id: &str) {
 	let code = black_box(
 		"606060405260005b620042408112156019575b6001016007565b600081905550600680602b6000396000f3606060405200".from_hex().unwrap()
 	);
 
-	b.iter(|| {
+	c.bench_function(bench_id, move |b| {
-		let mut params = ActionParams::default();
+		b.iter(|| {
-		params.gas = gas;
+			let mut params = ActionParams::default();
-		params.code = Some(Arc::new(code.clone()));
+			params.gas = gas;
+			params.code = Some(Arc::new(code.clone()));
 
-		let mut ext = FakeExt::new();
+			let mut ext = FakeExt::new();
-		let evm = Factory::default().create(params, ext.schedule(), ext.depth());
+			let evm = Factory::default().create(params, ext.schedule(), ext.depth());
-		let _ = evm.exec(&mut ext);
+			let _ = evm.exec(&mut ext);
+		})
 	});
 }
 
-#[bench]
+fn bench_rng_usize(c: &mut Criterion) {
-fn rng_usize(b: &mut Bencher) {
+	rng(U256::from(::std::usize::MAX), c, "rng_usize")
-	rng(U256::from(::std::usize::MAX), b)
 }
 
-#[bench]
+fn bench_rng_u256(c: &mut Criterion) {
-fn rng_u256(b: &mut Bencher) {
+	rng(!U256::zero(), c, "rng_u256")
-	rng(!U256::zero(), b)
 }
 
-fn rng(gas: U256, b: &mut Bencher) {
+fn rng(gas: U256, c: &mut Criterion, bench_id: &str) {
 	let code = black_box(
 		"6060604052600360056007600b60005b62004240811215607f5767ffe7649d5eca84179490940267f47ed85c4b9a6379019367f8e5dd9a5c994bba9390930267f91d87e4b8b74e55019267ff97f6f3b29cda529290920267f393ada8dd75c938019167fe8d437c45bb3735830267f47d9a7b5428ffec019150600101600f565b838518831882186000555050505050600680609a6000396000f3606060405200".from_hex().unwrap()
 	);
 
-	b.iter(|| {
+	c.bench_function(bench_id, move |b| {
-		let mut params = ActionParams::default();
+		b.iter(|| {
-		params.gas = gas;
+			let mut params = ActionParams::default();
-		params.code = Some(Arc::new(code.clone()));
+			params.gas = gas;
+			params.code = Some(Arc::new(code.clone()));
 
-		let mut ext = FakeExt::new();
+			let mut ext = FakeExt::new();
-		let evm = Factory::default().create(params, ext.schedule(), ext.depth());
+			let evm = Factory::default().create(params, ext.schedule(), ext.depth());
-		let _ = evm.exec(&mut ext);
+			let _ = evm.exec(&mut ext);
+		})
 	});
 }

util/blooms-db/Cargo.toml

@@ -10,4 +10,9 @@ ethbloom = "0.6.4"
 parking_lot = "0.8"
 
 [dev-dependencies]
+criterion = "0.3"
 tempdir = "0.3"
+
+[[bench]]
+name = "blooms"
+harness = false

util/blooms-db/benches/blooms.rs

@@ -14,21 +14,27 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity Ethereum.  If not, see <http://www.gnu.org/licenses/>.
 
-#![feature(test)]
+#[macro_use]
-
+extern crate criterion;
-extern crate test;
 extern crate tempdir;
 extern crate blooms_db;
 extern crate ethbloom;
 
 use std::iter;
-use test::Bencher;
+use criterion::Criterion;
 use tempdir::TempDir;
 use blooms_db::Database;
 use ethbloom::Bloom;
 
-#[bench]
+criterion_group!(
-fn blooms_filter_1_million_ok(b: &mut Bencher) {
+	blooms,
+	bench_blooms_filter_1_million_ok,
+	bench_blooms_filter_1_million_miss,
+	bench_blooms_filter_1_million_miss_and_ok,
+);
+criterion_main!(blooms);
+
+fn bench_blooms_filter_1_million_ok(c: &mut Criterion) {
 	let tempdir = TempDir::new("").unwrap();
 	let database = Database::open(tempdir.path()).unwrap();
 	database.insert_blooms(999_999, iter::once(&Bloom::zero())).unwrap();
@@ -38,14 +44,15 @@ fn blooms_filter_1_million_ok(b: &mut Bencher) {
 	database.insert_blooms(600_000, iter::once(&bloom)).unwrap();
 	database.insert_blooms(800_000, iter::once(&bloom)).unwrap();
 
-	b.iter(|| {
+	c.bench_function("blooms_filter_1_million_ok", move |b| {
-		let matches = database.filter(0, 999_999, Some(&bloom)).unwrap();
+		b.iter(|| {
-		assert_eq!(matches, vec![200_000, 400_000, 600_000, 800_000]);
+			let matches = database.filter(0, 999_999, Some(&bloom)).unwrap();
+			assert_eq!(matches, vec![200_000, 400_000, 600_000, 800_000]);
+		})
 	});
 }
 
-#[bench]
+fn bench_blooms_filter_1_million_miss(c: &mut Criterion) {
-fn blooms_filter_1_million_miss(b: &mut Bencher) {
 	let tempdir = TempDir::new("").unwrap();
 	let database = Database::open(tempdir.path()).unwrap();
 	database.insert_blooms(999_999, iter::once(&Bloom::zero())).unwrap();
@@ -56,14 +63,15 @@ fn blooms_filter_1_million_miss(b: &mut Bencher) {
 	database.insert_blooms(600_000, iter::once(&bloom)).unwrap();
 	database.insert_blooms(800_000, iter::once(&bloom)).unwrap();
 
-	b.iter(|| {
+	c.bench_function("blooms_filter_1_million_miss", move |b| {
-		let matches = database.filter(0, 999_999, Some(&bad_bloom)).unwrap();
+		b.iter(|| {
-		assert_eq!(matches, vec![200_000, 400_000, 600_000, 800_000]);
+			let matches = database.filter(0, 999_999, Some(&bad_bloom)).unwrap();
+			assert_eq!(matches, vec![200_000, 400_000, 600_000, 800_000]);
+		})
 	});
 }
 
-#[bench]
+fn bench_blooms_filter_1_million_miss_and_ok(c: &mut Criterion) {
-fn blooms_filter_1_million_miss_and_ok(b: &mut Bencher) {
 	let tempdir = TempDir::new("").unwrap();
 	let database = Database::open(tempdir.path()).unwrap();
 	database.insert_blooms(999_999, iter::once(&Bloom::zero())).unwrap();
@@ -74,8 +82,10 @@ fn blooms_filter_1_million_miss_and_ok(b: &mut Bencher) {
 	database.insert_blooms(600_000, iter::once(&bloom)).unwrap();
 	database.insert_blooms(800_000, iter::once(&bloom)).unwrap();
 
-	b.iter(|| {
+	c.bench_function("blooms_filter_1_million_miss_and_ok", move |b| {
-		let matches = database.filter(0, 999_999, &vec![bad_bloom, bloom]).unwrap();
+		b.iter(|| {
-		assert_eq!(matches, vec![200_000, 400_000, 600_000, 800_000]);
+			let matches = database.filter(0, 999_999, &vec![bad_bloom, bloom]).unwrap();
+			assert_eq!(matches, vec![200_000, 400_000, 600_000, 800_000]);
+		})
 	});
 }