ethash: implement Progpow (#9762)

* ethash: initial implementation of progpow

* progpow: use wrapping arithmetic

* progpow: cleanup comments

* progpow: fix keccak_f800

* progpow: reorder definitions

* progpow: general fixing

* progpow: add basic tests from geth

* progpow: generate c_dag and add test

* progpow: fix progpow_init and progpow_loop

* progpow: fix and add new test

* progpow: tabify

* progpow: add shared testvectors from geth and aleth

* progpow: add benchmarks

* progpow: don't read bytes from dag

* ethash: use criterion for progpow benchmarks

* progpow: dont borrow hash on fnv1a_hash

* progpow: don't borrow operand on progpow merge

* progpow: hardcode dag lookup function

we only support light verification anyway

* progpow: read double words directly from the dag

* progpow: inline some small functions

* progpow: remove some bounds checking from the main loop

* progpow: remove unreachable match cases

* progpow: remove bounds check in keccak_f800_round

* progpow: fix ptr::swap

* progpow: force loop unroll in keccak_f800_round

* progpow: remove unnecessary branching in progpow_loop

* progpow: force loop unroll in fill_mix

* progpow: silence unused warning

* progpow: dont run last keccak_f800_round out of the loop

rustc generates the same assembly, it unrolls the loop

* progpow: fix output of keccak_f800_short

* ethcore: support progpow in ethash engine

* ethash: fix typo

* ethcore, ethash: fix tests

* json: fix ethash spec tests

* ethash: update quick_get_difficulty for progpow

* ethash: drop light cache on progpow transition block

* ethash: fix quick_get_difficulty tests

* progpow: update to spec v0.9.0

* progpow: update to spec v0.9.1

* progpow: update to spec v0.9.2

* ethash: rename progpow benchmarks

* fix Cargo.lock bad merge

* ethash: only export modules for benchmarks

* ethash: progpow: remove unsafe unchecked indexing

* ethash: create enum for pow algorithm

* ethash: box the progpow cdag

* ethash: skip slow progpow test vectors on ci

* ethash: don't skip progpow test vectors

they don't take too long when running in release mode which is the case
for CI.

* ethash: progpow: update copyright date

Co-Authored-By: andresilva <andre.beat@gmail.com>

* ethcore: remove verification of ci-skip-tests on non-test builds

Cargo.lock

@@ -671,6 +671,8 @@ dependencies = [
  "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "primal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 

Cargo.toml

@@ -91,7 +91,7 @@ default = ["accounts"]
 accounts = ["ethcore-accounts", "parity-rpc/accounts"]
 miner-debug = ["ethcore/miner-debug"]
 json-tests = ["ethcore/json-tests"]
-ci-skip-issue = ["ethcore/ci-skip-issue"]
+ci-skip-tests = ["ethcore/ci-skip-tests"]
 test-heavy = ["ethcore/test-heavy"]
 evm-debug = ["ethcore/evm-debug"]
 evm-debug-tests = ["ethcore/evm-debug-tests"]

ethash/Cargo.toml

@@ -14,9 +14,19 @@ parking_lot = "0.7"
 primal = "0.2.3"
 
 [dev-dependencies]
-tempdir = "0.3"
 criterion = "0.2"
+rustc-hex = "1.0"
+serde_json = "1.0"
+tempdir = "0.3"
+
+[features]
+default = []
+bench = []
 
 [[bench]]
 name = "basic"
 harness = false
+
+[[bench]]
+name = "progpow"
+harness = false

ethash/benches/basic.rs

@@ -40,28 +40,28 @@ criterion_main!(basic);
 fn bench_light_compute_memmap(b: &mut Criterion) {
 	use std::env;
 
-	let builder = NodeCacheBuilder::new(OptimizeFor::Memory);
+	let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
 	let light = builder.light(&env::temp_dir(), 486382);
 
-	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| light.compute(&HASH, NONCE)));
+	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| light.compute(&HASH, NONCE, u64::max_value())));
 }
 
 fn bench_light_compute_memory(b: &mut Criterion) {
 	use std::env;
 
-	let builder = NodeCacheBuilder::new(OptimizeFor::Cpu);
+	let builder = NodeCacheBuilder::new(OptimizeFor::Cpu, u64::max_value());
 	let light = builder.light(&env::temp_dir(), 486382);
 
-	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| light.compute(&HASH, NONCE)));
+	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| light.compute(&HASH, NONCE, u64::max_value())));
 }
 
 fn bench_light_new_round_trip_memmap(b: &mut Criterion) {
 	use std::env;
 
 	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| {
-		let builder = NodeCacheBuilder::new(OptimizeFor::Memory);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
 		let light = builder.light(&env::temp_dir(), 486382);
-		light.compute(&HASH, NONCE);
+		light.compute(&HASH, NONCE, u64::max_value());
 	}));
 }
 
@@ -69,9 +69,9 @@ fn bench_light_new_round_trip_memory(b: &mut Criterion) {
 	use std::env;
 
 	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| {
-		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu, u64::max_value());
 		let light = builder.light(&env::temp_dir(), 486382);
-		light.compute(&HASH, NONCE);
+		light.compute(&HASH, NONCE, u64::max_value());
 	}));
 }
 
@@ -81,15 +81,15 @@ fn bench_light_from_file_round_trip_memory(b: &mut Criterion) {
 	let dir = env::temp_dir();
 	let height = 486382;
 	{
-		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu, u64::max_value());
 		let mut dummy = builder.light(&dir, height);
 		dummy.to_file().unwrap();
 	}
 
 	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| {
-		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Cpu, u64::max_value());
 		let light = builder.light_from_file(&dir, 486382).unwrap();
-		light.compute(&HASH, NONCE);
+		light.compute(&HASH, NONCE, u64::max_value());
 	}));
 }
 
@@ -100,14 +100,14 @@ fn bench_light_from_file_round_trip_memmap(b: &mut Criterion) {
 	let height = 486382;
 
 	{
-		let builder = NodeCacheBuilder::new(OptimizeFor::Memory);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
 		let mut dummy = builder.light(&dir, height);
 		dummy.to_file().unwrap();
 	}
 
 	b.bench_function("bench_light_compute_memmap", move |b| b.iter(|| {
-		let builder = NodeCacheBuilder::new(OptimizeFor::Memory);
+		let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
 		let light = builder.light_from_file(&dir, 486382).unwrap();
-		light.compute(&HASH, NONCE);
+		light.compute(&HASH, NONCE, u64::max_value());
 	}));
 }

ethash/benches/progpow.rs

@@ -0,0 +1,86 @@
+#[macro_use]
+extern crate criterion;
+extern crate ethash;
+extern crate rustc_hex;
+extern crate tempdir;
+
+use criterion::Criterion;
+use ethash::progpow;
+
+use tempdir::TempDir;
+use rustc_hex::FromHex;
+use ethash::{NodeCacheBuilder, OptimizeFor};
+use ethash::compute::light_compute;
+
+fn bench_hashimoto_light(c: &mut Criterion) {
+	let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
+	let tempdir = TempDir::new("").unwrap();
+	let light = builder.light(&tempdir.path(), 1);
+	let h = FromHex::from_hex("c9149cc0386e689d789a1c2f3d5d169a61a6218ed30e74414dc736e442ef3d1f").unwrap();
+	let mut hash = [0; 32];
+	hash.copy_from_slice(&h);
+
+	c.bench_function("hashimoto_light", move |b| {
+		b.iter(|| light_compute(&light, &hash, 0))
+	});
+}
+
+fn bench_progpow_light(c: &mut Criterion) {
+	let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
+	let tempdir = TempDir::new("").unwrap();
+	let cache = builder.new_cache(tempdir.into_path(), 0);
+
+	let h = FromHex::from_hex("c9149cc0386e689d789a1c2f3d5d169a61a6218ed30e74414dc736e442ef3d1f").unwrap();
+	let mut hash = [0; 32];
+	hash.copy_from_slice(&h);
+
+	c.bench_function("progpow_light", move |b| {
+		b.iter(|| {
+			let c_dag = progpow::generate_cdag(cache.as_ref());
+			progpow::progpow(
+				hash,
+				0,
+				0,
+				cache.as_ref(),
+				&c_dag,
+			);
+		})
+	});
+}
+
+fn bench_progpow_optimal_light(c: &mut Criterion) {
+	let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
+	let tempdir = TempDir::new("").unwrap();
+	let cache = builder.new_cache(tempdir.into_path(), 0);
+	let c_dag = progpow::generate_cdag(cache.as_ref());
+
+	let h = FromHex::from_hex("c9149cc0386e689d789a1c2f3d5d169a61a6218ed30e74414dc736e442ef3d1f").unwrap();
+	let mut hash = [0; 32];
+	hash.copy_from_slice(&h);
+
+	c.bench_function("progpow_optimal_light", move |b| {
+		b.iter(|| {
+			progpow::progpow(
+				hash,
+				0,
+				0,
+				cache.as_ref(),
+				&c_dag,
+			);
+		})
+	});
+}
+
+fn bench_keccak_f800_long(c: &mut Criterion) {
+	c.bench_function("keccak_f800_long(0, 0, 0)", |b| {
+		b.iter(|| progpow::keccak_f800_long([0; 32], 0, [0; 8]))
+	});
+}
+
+criterion_group!(benches,
+	bench_hashimoto_light,
+	bench_progpow_light,
+	bench_progpow_optimal_light,
+	bench_keccak_f800_long,
+);
+criterion_main!(benches);

ethash/res/progpow_testvectors.json

@@ -0,0 +1,86 @@
+[
+  [
+    0,
+    "0000000000000000000000000000000000000000000000000000000000000000",
+    "0000000000000000",
+    "faeb1be51075b03a4ff44b335067951ead07a3b078539ace76fd56fc410557a3",
+    "63155f732f2bf556967f906155b510c917e48e99685ead76ea83f4eca03ab12b"
+  ],
+  [
+    49,
+    "63155f732f2bf556967f906155b510c917e48e99685ead76ea83f4eca03ab12b",
+    "0000000006ff2c47",
+    "c789c1180f890ec555ff42042913465481e8e6bc512cb981e1c1108dc3f2227d",
+    "9e7248f20914913a73d80a70174c331b1d34f260535ac3631d770e656b5dd922"
+  ],
+  [
+    50,
+    "9e7248f20914913a73d80a70174c331b1d34f260535ac3631d770e656b5dd922",
+    "00000000076e482e",
+    "c7340542c2a06b3a7dc7222635f7cd402abf8b528ae971ddac6bbe2b0c7cb518",
+    "de37e1824c86d35d154cf65a88de6d9286aec4f7f10c3fc9f0fa1bcc2687188d"
+  ],
+  [
+    99,
+    "de37e1824c86d35d154cf65a88de6d9286aec4f7f10c3fc9f0fa1bcc2687188d",
+    "000000003917afab",
+    "f5e60b2c5bfddd136167a30cbc3c8dbdbd15a512257dee7964e0bc6daa9f8ba7",
+    "ac7b55e801511b77e11d52e9599206101550144525b5679f2dab19386f23dcce"
+  ],
+  [
+    29950,
+    "ac7b55e801511b77e11d52e9599206101550144525b5679f2dab19386f23dcce",
+    "005d409dbc23a62a",
+    "07393d15805eb08ee6fc6cb3ad4ad1010533bd0ff92d6006850246829f18fd6e",
+    "e43d7e0bdc8a4a3f6e291a5ed790b9fa1a0948a2b9e33c844888690847de19f5"
+  ],
+  [
+    29999,
+    "e43d7e0bdc8a4a3f6e291a5ed790b9fa1a0948a2b9e33c844888690847de19f5",
+    "005db5fa4c2a3d03",
+    "7551bddf977491da2f6cfc1679299544b23483e8f8ee0931c4c16a796558a0b8",
+    "d34519f72c97cae8892c277776259db3320820cb5279a299d0ef1e155e5c6454"
+  ],
+  [
+    30000,
+    "d34519f72c97cae8892c277776259db3320820cb5279a299d0ef1e155e5c6454",
+    "005db8607994ff30",
+    "f1c2c7c32266af9635462e6ce1c98ebe4e7e3ecab7a38aaabfbf2e731e0fbff4",
+    "8b6ce5da0b06d18db7bd8492d9e5717f8b53e7e098d9fef7886d58a6e913ef64"
+  ],
+  [
+    30049,
+    "8b6ce5da0b06d18db7bd8492d9e5717f8b53e7e098d9fef7886d58a6e913ef64",
+    "005e2e215a8ca2e7",
+    "57fe6a9fbf920b4e91deeb66cb0efa971e08229d1a160330e08da54af0689add",
+    "c2c46173481b9ced61123d2e293b42ede5a1b323210eb2a684df0874ffe09047"
+  ],
+  [
+    30050,
+    "c2c46173481b9ced61123d2e293b42ede5a1b323210eb2a684df0874ffe09047",
+    "005e30899481055e",
+    "ba30c61cc5a2c74a5ecaf505965140a08f24a296d687e78720f0b48baf712f2d",
+    "ea42197eb2ba79c63cb5e655b8b1f612c5f08aae1a49ff236795a3516d87bc71"
+  ],
+  [
+    30099,
+    "ea42197eb2ba79c63cb5e655b8b1f612c5f08aae1a49ff236795a3516d87bc71",
+    "005ea6aef136f88b",
+    "cfd5e46048cd133d40f261fe8704e51d3f497fc14203ac6a9ef6a0841780b1cd",
+    "49e15ba4bf501ce8fe8876101c808e24c69a859be15de554bf85dbc095491bd6"
+  ],
+  [
+    59950,
+    "49e15ba4bf501ce8fe8876101c808e24c69a859be15de554bf85dbc095491bd6",
+    "02ebe0503bd7b1da",
+    "21511fbaa31fb9f5fc4998a754e97b3083a866f4de86fa7500a633346f56d773",
+    "f5c50ba5c0d6210ddb16250ec3efda178de857b2b1703d8d5403bd0f848e19cf"
+  ],
+  [
+    59999,
+    "f5c50ba5c0d6210ddb16250ec3efda178de857b2b1703d8d5403bd0f848e19cf",
+    "02edb6275bd221e3",
+    "653eda37d337e39d311d22be9bbd3458d3abee4e643bee4a7280a6d08106ef98",
+    "341562d10d4afb706ec2c8d5537cb0c810de02b4ebb0a0eea5ae335af6fb2e88"
+  ]
+]

ethash/src/cache.rs

@@ -69,6 +69,7 @@ pub struct NodeCacheBuilder {
 	// TODO: Remove this locking and just use an `Rc`?
 	seedhash: Arc<Mutex<SeedHashCompute>>,
 	optimize_for: OptimizeFor,
+	progpow_transition: u64,
 }
 
 // TODO: Abstract the "optimize for" logic
@@ -82,17 +83,18 @@ pub struct NodeCache {
 
 impl NodeCacheBuilder {
 	pub fn light(&self, cache_dir: &Path, block_number: u64) -> Light {
-		Light::new_with_builder(self, cache_dir, block_number)
+		Light::new_with_builder(self, cache_dir, block_number, self.progpow_transition)
 	}
 
 	pub fn light_from_file(&self, cache_dir: &Path, block_number: u64) -> io::Result<Light> {
-		Light::from_file_with_builder(self, cache_dir, block_number)
+		Light::from_file_with_builder(self, cache_dir, block_number, self.progpow_transition)
 	}
 
-	pub fn new<T: Into<Option<OptimizeFor>>>(optimize_for: T) -> Self {
+	pub fn new<T: Into<Option<OptimizeFor>>>(optimize_for: T, progpow_transition: u64) -> Self {
 		NodeCacheBuilder {
 			seedhash: Arc::new(Mutex::new(SeedHashCompute::default())),
 			optimize_for: optimize_for.into().unwrap_or_default(),
+			progpow_transition
 		}
 	}
 

ethash/src/compute.rs

@@ -21,6 +21,7 @@
 
 use keccak::{keccak_512, keccak_256, H256};
 use cache::{NodeCache, NodeCacheBuilder};
+use progpow::{CDag, generate_cdag, progpow, keccak_f800_short, keccak_f800_long};
 use seed_compute::SeedHashCompute;
 use shared::*;
 use std::io;
@@ -30,7 +31,7 @@ use std::path::Path;
 
 const MIX_WORDS: usize = ETHASH_MIX_BYTES / 4;
 const MIX_NODES: usize = MIX_WORDS / NODE_WORDS;
-const FNV_PRIME: u32 = 0x01000193;
+pub const FNV_PRIME: u32 = 0x01000193;
 
 /// Computation result
 pub struct ProofOfWork {
@@ -40,9 +41,15 @@ pub struct ProofOfWork {
 	pub mix_hash: H256,
 }
 
+enum Algorithm {
+	Hashimoto,
+	Progpow(Box<CDag>),
+}
+
 pub struct Light {
 	block_number: u64,
 	cache: NodeCache,
+	algorithm: Algorithm,
 }
 
 /// Light cache structure
@@ -51,32 +58,55 @@ impl Light {
 		builder: &NodeCacheBuilder,
 		cache_dir: &Path,
 		block_number: u64,
+		progpow_transition: u64,
 	) -> Self {
 		let cache = builder.new_cache(cache_dir.to_path_buf(), block_number);
 
-		Light {
+		let algorithm = if block_number >= progpow_transition {
-			block_number: block_number,
+			Algorithm::Progpow(Box::new(generate_cdag(cache.as_ref())))
-			cache: cache,
+		} else {
-		}
+			Algorithm::Hashimoto
+		};
+
+		Light { block_number, cache, algorithm }
 	}
 
 	/// Calculate the light boundary data
 	/// `header_hash` - The header hash to pack into the mix
 	/// `nonce` - The nonce to pack into the mix
-	pub fn compute(&self, header_hash: &H256, nonce: u64) -> ProofOfWork {
+	pub fn compute(&self, header_hash: &H256, nonce: u64, block_number: u64) -> ProofOfWork {
-		light_compute(self, header_hash, nonce)
+		match self.algorithm {
+			Algorithm::Progpow(ref c_dag) => {
+				let (value, mix_hash) = progpow(
+					*header_hash,
+					nonce,
+					block_number,
+					self.cache.as_ref(),
+					c_dag,
+				);
+
+				ProofOfWork { value, mix_hash }
+			},
+			Algorithm::Hashimoto => light_compute(self, header_hash, nonce),
+		}
+
 	}
 
 	pub fn from_file_with_builder(
 		builder: &NodeCacheBuilder,
 		cache_dir: &Path,
 		block_number: u64,
+		progpow_transition: u64,
 	) -> io::Result<Self> {
 		let cache = builder.from_file(cache_dir.to_path_buf(), block_number)?;
-		Ok(Light {
+
-			block_number: block_number,
+		let algorithm = if block_number >= progpow_transition {
-			cache: cache,
+			Algorithm::Progpow(Box::new(generate_cdag(cache.as_ref())))
-		})
+		} else {
+			Algorithm::Hashimoto
+		};
+
+		Ok(Light { block_number, cache, algorithm })
 	}
 
 	pub fn to_file(&mut self) -> io::Result<&Path> {
@@ -99,27 +129,32 @@ fn fnv_hash(x: u32, y: u32) -> u32 {
 /// `nonce`            The block's nonce
 /// `mix_hash`         The mix digest hash
 /// Boundary recovered from mix hash
-pub fn quick_get_difficulty(header_hash: &H256, nonce: u64, mix_hash: &H256) -> H256 {
+pub fn quick_get_difficulty(header_hash: &H256, nonce: u64, mix_hash: &H256, progpow: bool) -> H256 {
 	unsafe {
-		// This is safe - the `keccak_512` call below reads the first 40 bytes (which we explicitly set
+		if progpow {
-		// with two `copy_nonoverlapping` calls) but writes the first 64, and then we explicitly write
+			let seed = keccak_f800_short(*header_hash, nonce, [0u32; 8]);
-		// the next 32 bytes before we read the whole thing with `keccak_256`.
+			keccak_f800_long(*header_hash, seed, mem::transmute(*mix_hash))
-		//
+		} else {
-		// This cannot be elided by the compiler as it doesn't know the implementation of
+			// This is safe - the `keccak_512` call below reads the first 40 bytes (which we explicitly set
-		// `keccak_512`.
+			// with two `copy_nonoverlapping` calls) but writes the first 64, and then we explicitly write
-		let mut buf: [u8; 64 + 32] = mem::uninitialized();
+			// the next 32 bytes before we read the whole thing with `keccak_256`.
+			//
+			// This cannot be elided by the compiler as it doesn't know the implementation of
+			// `keccak_512`.
+			let mut buf: [u8; 64 + 32] = mem::uninitialized();
 
-		ptr::copy_nonoverlapping(header_hash.as_ptr(), buf.as_mut_ptr(), 32);
+			ptr::copy_nonoverlapping(header_hash.as_ptr(), buf.as_mut_ptr(), 32);
-		ptr::copy_nonoverlapping(&nonce as *const u64 as *const u8, buf[32..].as_mut_ptr(), 8);
+			ptr::copy_nonoverlapping(&nonce as *const u64 as *const u8, buf[32..].as_mut_ptr(), 8);
 
-		keccak_512::unchecked(buf.as_mut_ptr(), 64, buf.as_ptr(), 40);
+			keccak_512::unchecked(buf.as_mut_ptr(), 64, buf.as_ptr(), 40);
-		ptr::copy_nonoverlapping(mix_hash.as_ptr(), buf[64..].as_mut_ptr(), 32);
+			ptr::copy_nonoverlapping(mix_hash.as_ptr(), buf[64..].as_mut_ptr(), 32);
 
-		// This is initialized in `keccak_256`
+			// This is initialized in `keccak_256`
-		let mut hash: [u8; 32] = mem::uninitialized();
+			let mut hash: [u8; 32] = mem::uninitialized();
-		keccak_256::unchecked(hash.as_mut_ptr(), hash.len(), buf.as_ptr(), buf.len());
+			keccak_256::unchecked(hash.as_mut_ptr(), hash.len(), buf.as_ptr(), buf.len());
 
-		hash
+			hash
+		}
 	}
 }
 
@@ -272,7 +307,7 @@ fn hash_compute(light: &Light, full_size: usize, header_hash: &H256, nonce: u64)
 		// We overwrite the second half since `keccak_256` has an internal buffer and so allows
 		// overlapping arrays as input.
 		let write_ptr: *mut u8 = &mut buf.compress_bytes as *mut [u8; 32] as *mut u8;
-		unsafe { 
+		unsafe {
 			keccak_256::unchecked(
 				write_ptr,
 				buf.compress_bytes.len(),
@@ -287,7 +322,7 @@ fn hash_compute(light: &Light, full_size: usize, header_hash: &H256, nonce: u64)
 }
 
 // TODO: Use the `simd` crate
-fn calculate_dag_item(node_index: u32, cache: &[Node]) -> Node {
+pub fn calculate_dag_item(node_index: u32, cache: &[Node]) -> Node {
 	let num_parent_nodes = cache.len();
 	let mut ret = cache[node_index as usize % num_parent_nodes].clone();
 	ret.as_words_mut()[0] ^= node_index;
@@ -361,13 +396,13 @@ mod test {
 			0x4a, 0x8e, 0x95, 0x69, 0xef, 0xc7, 0xd7, 0x1b, 0x33, 0x35, 0xdf, 0x36, 0x8c, 0x9a,
 			0xe9, 0x7e, 0x53, 0x84,
 		];
-		assert_eq!(quick_get_difficulty(&hash, nonce, &mix_hash)[..], boundary_good[..]);
+		assert_eq!(quick_get_difficulty(&hash, nonce, &mix_hash, false)[..], boundary_good[..]);
 		let boundary_bad = [
 			0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x3a, 0x9b, 0x6c, 0x69, 0xbc, 0x2c, 0xe2, 0xa2,
 			0x4a, 0x8e, 0x95, 0x69, 0xef, 0xc7, 0xd7, 0x1b, 0x33, 0x35, 0xdf, 0x36, 0x8c, 0x9a,
 			0xe9, 0x7e, 0x53, 0x84,
 		];
-		assert!(quick_get_difficulty(&hash, nonce, &mix_hash)[..] != boundary_bad[..]);
+		assert!(quick_get_difficulty(&hash, nonce, &mix_hash, false)[..] != boundary_bad[..]);
 	}
 
 	#[test]
@@ -391,7 +426,7 @@ mod test {
 
 		let tempdir = TempDir::new("").unwrap();
 		// difficulty = 0x085657254bd9u64;
-		let light = NodeCacheBuilder::new(None).light(tempdir.path(), 486382);
+		let light = NodeCacheBuilder::new(None, u64::max_value()).light(tempdir.path(), 486382);
 		let result = light_compute(&light, &hash, nonce);
 		assert_eq!(result.mix_hash[..], mix_hash[..]);
 		assert_eq!(result.value[..], boundary[..]);
@@ -400,7 +435,7 @@ mod test {
 	#[test]
 	fn test_drop_old_data() {
 		let tempdir = TempDir::new("").unwrap();
-		let builder = NodeCacheBuilder::new(None);
+		let builder = NodeCacheBuilder::new(None, u64::max_value());
 		let first = builder.light(tempdir.path(), 0).to_file().unwrap().to_owned();
 
 		let second = builder.light(tempdir.path(), ETHASH_EPOCH_LENGTH).to_file().unwrap().to_owned();

ethash/src/lib.rs

@@ -25,15 +25,30 @@ extern crate crunchy;
 #[macro_use]
 extern crate log;
 
+#[cfg(test)]
+extern crate rustc_hex;
+
+#[cfg(test)]
+extern crate serde_json;
+
 #[cfg(test)]
 extern crate tempdir;
 
+#[cfg(feature = "bench")]
+pub mod compute;
+#[cfg(not(feature = "bench"))]
 mod compute;
+
 mod seed_compute;
 mod cache;
 mod keccak;
 mod shared;
 
+#[cfg(feature = "bench")]
+pub mod progpow;
+#[cfg(not(feature = "bench"))]
+mod progpow;
+
 pub use cache::{NodeCacheBuilder, OptimizeFor};
 pub use compute::{ProofOfWork, quick_get_difficulty, slow_hash_block_number};
 use compute::Light;
@@ -59,14 +74,16 @@ pub struct EthashManager {
 	nodecache_builder: NodeCacheBuilder,
 	cache: Mutex<LightCache>,
 	cache_dir: PathBuf,
+	progpow_transition: u64,
 }
 
 impl EthashManager {
 	/// Create a new new instance of ethash manager
-	pub fn new<T: Into<Option<OptimizeFor>>>(cache_dir: &Path, optimize_for: T) -> EthashManager {
+	pub fn new<T: Into<Option<OptimizeFor>>>(cache_dir: &Path, optimize_for: T, progpow_transition: u64) -> EthashManager {
 		EthashManager {
 			cache_dir: cache_dir.to_path_buf(),
-			nodecache_builder: NodeCacheBuilder::new(optimize_for.into().unwrap_or_default()),
+			nodecache_builder: NodeCacheBuilder::new(optimize_for.into().unwrap_or_default(), progpow_transition),
+			progpow_transition: progpow_transition,
 			cache: Mutex::new(LightCache {
 				recent_epoch: None,
 				recent: None,
@@ -85,27 +102,33 @@ impl EthashManager {
 		let epoch = block_number / ETHASH_EPOCH_LENGTH;
 		let light = {
 			let mut lights = self.cache.lock();
-			let light = match lights.recent_epoch.clone() {
+			let light = if block_number == self.progpow_transition {
-				Some(ref e) if *e == epoch => lights.recent.clone(),
+				// we need to regenerate the cache to trigger algorithm change to progpow inside `Light`
-				_ => match lights.prev_epoch.clone() {
+				None
-					Some(e) if e == epoch => {
+			} else {
-						// don't swap if recent is newer.
+				match lights.recent_epoch.clone() {
-						if lights.recent_epoch > lights.prev_epoch {
+					Some(ref e) if *e == epoch => lights.recent.clone(),
-							None
+					_ => match lights.prev_epoch.clone() {
-						} else {
+						Some(e) if e == epoch => {
-							// swap
+							// don't swap if recent is newer.
-							let t = lights.prev_epoch;
+							if lights.recent_epoch > lights.prev_epoch {
-							lights.prev_epoch = lights.recent_epoch;
+								None
-							lights.recent_epoch = t;
+							} else {
-							let t = lights.prev.clone();
+								// swap
-							lights.prev = lights.recent.clone();
+								let t = lights.prev_epoch;
-							lights.recent = t;
+								lights.prev_epoch = lights.recent_epoch;
-							lights.recent.clone()
+								lights.recent_epoch = t;
+								let t = lights.prev.clone();
+								lights.prev = lights.recent.clone();
+								lights.recent = t;
+								lights.recent.clone()
+							}
 						}
-					}
+						_ => None,
-					_ => None,
+					},
-				},
+				}
 			};
+
 			match light {
 				None => {
 					let light = match self.nodecache_builder.light_from_file(
@@ -132,7 +155,7 @@ impl EthashManager {
 				Some(light) => light,
 			}
 		};
-		light.compute(header_hash, nonce)
+		light.compute(header_hash, nonce, block_number)
 	}
 }
 
@@ -164,7 +187,7 @@ fn test_lru() {
 	use tempdir::TempDir;
 
 	let tempdir = TempDir::new("").unwrap();
-	let ethash = EthashManager::new(tempdir.path(), None);
+	let ethash = EthashManager::new(tempdir.path(), None, u64::max_value());
 	let hash = [0u8; 32];
 	ethash.compute_light(1, &hash, 1);
 	ethash.compute_light(50000, &hash, 1);

ethash/src/progpow.rs

@@ -0,0 +1,595 @@
+// Copyright 2015-2019 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use compute::{FNV_PRIME, calculate_dag_item};
+use keccak::H256;
+use shared::{ETHASH_ACCESSES, ETHASH_MIX_BYTES, Node, get_data_size};
+
+const PROGPOW_CACHE_BYTES: usize = 16 * 1024;
+const PROGPOW_CACHE_WORDS: usize = PROGPOW_CACHE_BYTES / 4;
+const PROGPOW_CNT_CACHE: usize = 12;
+const PROGPOW_CNT_MATH: usize = 20;
+const PROGPOW_CNT_DAG: usize = ETHASH_ACCESSES;
+const PROGPOW_DAG_LOADS: usize = 4;
+const PROGPOW_MIX_BYTES: usize = 2 * ETHASH_MIX_BYTES;
+const PROGPOW_PERIOD_LENGTH: usize = 50; // blocks per progpow epoch (N)
+const PROGPOW_LANES: usize = 16;
+const PROGPOW_REGS: usize = 32;
+
+const FNV_HASH: u32 = 0x811c9dc5;
+
+const KECCAKF_RNDC: [u32; 24] = [
+	0x00000001, 0x00008082, 0x0000808a, 0x80008000, 0x0000808b, 0x80000001,
+	0x80008081, 0x00008009, 0x0000008a, 0x00000088, 0x80008009, 0x8000000a,
+	0x8000808b, 0x0000008b, 0x00008089, 0x00008003, 0x00008002, 0x00000080,
+	0x0000800a, 0x8000000a, 0x80008081, 0x00008080, 0x80000001, 0x80008008
+];
+
+const KECCAKF_ROTC: [u32; 24] = [
+	1,  3,  6,  10, 15, 21, 28, 36, 45, 55, 2,  14,
+	27, 41, 56, 8,  25, 43, 62, 18, 39, 61, 20, 44
+];
+
+const KECCAKF_PILN: [usize; 24] = [
+	10, 7,  11, 17, 18, 3, 5,  16, 8,  21, 24, 4,
+	15, 23, 19, 13, 12, 2, 20, 14, 22, 9,  6,  1
+];
+
+fn keccak_f800_round(st: &mut [u32; 25], r: usize) {
+	// Theta
+	let mut bc = [0u32; 5];
+	for i in 0..bc.len() {
+		bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15] ^ st[i + 20];
+	}
+
+	for i in 0..bc.len() {
+		let t = bc[(i + 4) % 5] ^ bc[(i + 1) % 5].rotate_left(1);
+		for j in (0..st.len()).step_by(5) {
+			st[j + i] ^= t;
+		}
+	}
+
+	// Rho Pi
+	let mut t = st[1];
+
+	debug_assert_eq!(KECCAKF_ROTC.len(), 24);
+	for i in 0..24 {
+		let j = KECCAKF_PILN[i];
+		bc[0] = st[j];
+		st[j] = t.rotate_left(KECCAKF_ROTC[i]);
+		t = bc[0];
+	}
+
+	// Chi
+	for j in (0..st.len()).step_by(5) {
+		for i in 0..bc.len() {
+			bc[i] = st[j + i];
+		}
+		for i in 0..bc.len() {
+			st[j + i] ^= (!bc[(i + 1) % 5]) & bc[(i + 2) % 5];
+		}
+	}
+
+	// Iota
+	debug_assert!(r < KECCAKF_RNDC.len());
+	st[0] ^= KECCAKF_RNDC[r];
+}
+
+fn keccak_f800(header_hash: H256, nonce: u64, result: [u32; 8], st: &mut [u32; 25]) {
+	for i in 0..8 {
+		st[i] = (header_hash[4 * i] as u32) +
+			((header_hash[4 * i + 1] as u32) << 8) +
+			((header_hash[4 * i + 2] as u32) << 16) +
+			((header_hash[4 * i + 3] as u32) << 24);
+	}
+
+	st[8] = nonce as u32;
+	st[9] = (nonce >> 32) as u32;
+
+	for i in 0..8 {
+		st[10 + i] = result[i];
+	}
+
+	for r in 0..22 {
+		keccak_f800_round(st, r);
+	}
+}
+
+pub fn keccak_f800_short(header_hash: H256, nonce: u64, result: [u32; 8]) -> u64 {
+	let mut st = [0u32; 25];
+	keccak_f800(header_hash, nonce, result, &mut st);
+	(st[0].swap_bytes() as u64) << 32 | st[1].swap_bytes() as u64
+}
+
+pub fn keccak_f800_long(header_hash: H256, nonce: u64, result: [u32; 8]) -> H256 {
+	let mut st = [0u32; 25];
+	keccak_f800(header_hash, nonce, result, &mut st);
+
+	// NOTE: transmute from `[u32; 8]` to `[u8; 32]`
+	unsafe {
+		std::mem::transmute(
+			[st[0], st[1], st[2], st[3], st[4], st[5], st[6], st[7]]
+		)
+	}
+}
+
+#[inline]
+fn fnv1a_hash(h: u32, d: u32) -> u32 {
+	(h ^ d).wrapping_mul(FNV_PRIME)
+}
+
+#[derive(Clone)]
+struct Kiss99 {
+	z: u32,
+	w: u32,
+	jsr: u32,
+	jcong: u32,
+}
+
+impl Kiss99 {
+	fn new(z: u32, w: u32, jsr: u32, jcong: u32) -> Kiss99 {
+		Kiss99 { z, w, jsr, jcong }
+	}
+
+	#[inline]
+	fn next_u32(&mut self) -> u32 {
+		self.z = 36969u32.wrapping_mul(self.z & 65535).wrapping_add(self.z >> 16);
+		self.w = 18000u32.wrapping_mul(self.w & 65535).wrapping_add(self.w >> 16);
+		let mwc = (self.z << 16).wrapping_add(self.w);
+		self.jsr ^= self.jsr << 17;
+		self.jsr ^= self.jsr >> 13;
+		self.jsr ^= self.jsr << 5;
+		self.jcong = 69069u32.wrapping_mul(self.jcong).wrapping_add(1234567);
+
+		(mwc ^ self.jcong).wrapping_add(self.jsr)
+	}
+}
+
+fn fill_mix(seed: u64, lane_id: u32) -> [u32; PROGPOW_REGS] {
+	// Use FNV to expand the per-warp seed to per-lane
+	// Use KISS to expand the per-lane seed to fill mix
+	let z = fnv1a_hash(FNV_HASH, seed as u32);
+	let w = fnv1a_hash(z, (seed >> 32) as u32);
+	let jsr = fnv1a_hash(w, lane_id);
+	let jcong = fnv1a_hash(jsr, lane_id);
+
+	let mut rnd = Kiss99::new(z, w, jsr, jcong);
+
+	let mut mix = [0; PROGPOW_REGS];
+
+	debug_assert_eq!(PROGPOW_REGS, 32);
+	for i in 0..32 {
+		mix[i] = rnd.next_u32();
+	}
+
+	mix
+}
+
+// Merge new data from b into the value in a. Assuming A has high entropy only
+// do ops that retain entropy even if B is low entropy (IE don't do A&B)
+fn merge(a: u32, b: u32, r: u32) -> u32 {
+	match r % 4 {
+		0 => a.wrapping_mul(33).wrapping_add(b),
+		1 => (a ^ b).wrapping_mul(33),
+		2 => a.rotate_left(((r >> 16) % 31) + 1) ^ b,
+		_ => a.rotate_right(((r >> 16) % 31) + 1) ^ b,
+	}
+}
+
+fn math(a: u32, b: u32, r: u32) -> u32 {
+	match r % 11 {
+		0 => a.wrapping_add(b),
+		1 => a.wrapping_mul(b),
+		2 => ((a as u64).wrapping_mul(b as u64) >> 32) as u32,
+		3 => a.min(b),
+		4 => a.rotate_left(b),
+		5 => a.rotate_right(b),
+		6 => a & b,
+		7 => a | b,
+		8 => a ^ b,
+		9 => a.leading_zeros() + b.leading_zeros(),
+		_ => a.count_ones() + b.count_ones(),
+	}
+}
+
+fn progpow_init(seed: u64) -> (Kiss99, [u32; PROGPOW_REGS], [u32; PROGPOW_REGS]) {
+	let z = fnv1a_hash(FNV_HASH, seed as u32);
+	let w = fnv1a_hash(z, (seed >> 32) as u32);
+	let jsr = fnv1a_hash(w, seed as u32);
+	let jcong = fnv1a_hash(jsr, (seed >> 32) as u32);
+
+	let mut rnd = Kiss99::new(z, w, jsr, jcong);
+
+	// Create a random sequence of mix destinations for merge() and mix sources
+	// for cache reads guarantees every destination merged once and guarantees
+	// no duplicate cache reads, which could be optimized away. Uses
+	// Fisher-Yates shuffle.
+	let mut mix_seq_dst = [0u32; PROGPOW_REGS];
+	let mut mix_seq_cache = [0u32; PROGPOW_REGS];
+	for i in 0..mix_seq_dst.len() {
+		mix_seq_dst[i] = i as u32;
+		mix_seq_cache[i] = i as u32;
+	}
+
+	for i in (1..mix_seq_dst.len()).rev() {
+		let j = rnd.next_u32() as usize % (i + 1);
+		mix_seq_dst.swap(i, j);
+
+		let j = rnd.next_u32() as usize % (i + 1);
+		mix_seq_cache.swap(i, j);
+	}
+
+	(rnd, mix_seq_dst, mix_seq_cache)
+}
+
+pub type CDag = [u32; PROGPOW_CACHE_WORDS];
+
+fn progpow_loop(
+	seed: u64,
+	loop_: usize,
+	mix: &mut [[u32; PROGPOW_REGS]; PROGPOW_LANES],
+	cache: &[Node],
+	c_dag: &CDag,
+	data_size: usize,
+) {
+	// All lanes share a base address for the global load. Global offset uses
+	// mix[0] to guarantee it depends on the load result.
+	let g_offset = mix[loop_ % PROGPOW_LANES][0] as usize %
+		(64 * data_size / (PROGPOW_LANES * PROGPOW_DAG_LOADS));
+
+	// 256 bytes of dag data
+	let mut dag_item = [0u32; 64];
+
+	// Fetch DAG nodes (64 bytes each)
+	for l in 0..PROGPOW_DAG_LOADS {
+		let index = g_offset * PROGPOW_LANES * PROGPOW_DAG_LOADS + l * 16;
+		let node = calculate_dag_item(index as u32 / 16, cache);
+		dag_item[l * 16..(l + 1) * 16].clone_from_slice(node.as_words());
+	}
+
+	let (rnd, mix_seq_dst, mix_seq_cache) = progpow_init(seed);
+
+	// Lanes can execute in parallel and will be convergent
+	for l in 0..mix.len() {
+		let mut rnd = rnd.clone();
+
+		// Initialize the seed and mix destination sequence
+		let mut mix_seq_dst_cnt = 0;
+		let mut mix_seq_cache_cnt = 0;
+
+		let mut mix_dst = || {
+			let res = mix_seq_dst[mix_seq_dst_cnt % PROGPOW_REGS] as usize;
+			mix_seq_dst_cnt += 1;
+			res
+		};
+		let mut mix_cache = || {
+			let res = mix_seq_cache[mix_seq_cache_cnt % PROGPOW_REGS] as usize;
+			mix_seq_cache_cnt += 1;
+			res
+		};
+
+		for i in 0..PROGPOW_CNT_CACHE.max(PROGPOW_CNT_MATH) {
+			if i < PROGPOW_CNT_CACHE {
+				// Cached memory access, lanes access random 32-bit locations
+				// within the first portion of the DAG
+				let offset = mix[l][mix_cache()] as usize % PROGPOW_CACHE_WORDS;
+				let data = c_dag[offset];
+				let dst = mix_dst();
+
+				mix[l][dst] = merge(mix[l][dst], data, rnd.next_u32());
+			}
+
+			if i < PROGPOW_CNT_MATH {
+				// Random math
+				// Generate 2 unique sources
+				let src_rnd = rnd.next_u32() % (PROGPOW_REGS * (PROGPOW_REGS - 1)) as u32;
+				let src1 = src_rnd % PROGPOW_REGS as u32; // 0 <= src1 < PROGPOW_REGS
+				let mut src2 = src_rnd / PROGPOW_REGS as u32; // 0 <= src2 < PROGPOW_REGS - 1
+				if src2 >= src1 {
+					src2 += 1; // src2 is now any reg other than src1
+				}
+
+				let data = math(mix[l][src1 as usize], mix[l][src2 as usize], rnd.next_u32());
+				let dst = mix_dst();
+
+				mix[l][dst] = merge(mix[l][dst], data, rnd.next_u32());
+			}
+		}
+
+		// Global load to sequential locations
+		let mut data_g = [0u32; PROGPOW_DAG_LOADS];
+		let index = ((l ^ loop_) % PROGPOW_LANES) * PROGPOW_DAG_LOADS;
+		for i in 0..PROGPOW_DAG_LOADS {
+			data_g[i] = dag_item[index + i];
+		}
+
+		// Consume the global load data at the very end of the loop to allow
+		// full latency hiding. Always merge into `mix[0]` to feed the offset
+		// calculation.
+		mix[l][0] = merge(mix[l][0], data_g[0], rnd.next_u32());
+		for i in 1..PROGPOW_DAG_LOADS {
+			let dst = mix_dst();
+			mix[l][dst] = merge(mix[l][dst], data_g[i], rnd.next_u32());
+		}
+	}
+}
+
+pub fn progpow(
+	header_hash: H256,
+	nonce: u64,
+	block_number: u64,
+	cache: &[Node],
+	c_dag: &CDag,
+) -> (H256, H256) {
+	let mut mix = [[0u32; PROGPOW_REGS]; PROGPOW_LANES];
+	let mut lane_results = [0u32; PROGPOW_LANES];
+	let mut result = [0u32; 8];
+
+	let data_size = get_data_size(block_number) / PROGPOW_MIX_BYTES;
+
+	// NOTE: This assert is required to aid the optimizer elide the non-zero
+	// remainder check in `progpow_loop`.
+	assert!(data_size > 0);
+
+	// Initialize mix for all lanes
+	let seed = keccak_f800_short(header_hash, nonce, result);
+
+	for l in 0..mix.len() {
+		mix[l] = fill_mix(seed, l as u32);
+	}
+
+	// Execute the randomly generated inner loop
+	let period = block_number / PROGPOW_PERIOD_LENGTH as u64;
+	for i in 0..PROGPOW_CNT_DAG {
+		progpow_loop(
+			period,
+			i,
+			&mut mix,
+			cache,
+			c_dag,
+			data_size,
+		);
+	}
+
+	// Reduce mix data to a single per-lane result
+	for l in 0..lane_results.len() {
+		lane_results[l] = FNV_HASH;
+		for i in 0..PROGPOW_REGS {
+			lane_results[l] = fnv1a_hash(lane_results[l], mix[l][i]);
+		}
+	}
+
+	// Reduce all lanes to a single 128-bit result
+	result = [FNV_HASH; 8];
+	for l in 0..PROGPOW_LANES {
+		result[l % 8] = fnv1a_hash(result[l % 8], lane_results[l]);
+	}
+
+	let digest = keccak_f800_long(header_hash, seed, result);
+
+	// NOTE: transmute from `[u32; 8]` to `[u8; 32]`
+	let result = unsafe { ::std::mem::transmute(result) };
+
+	(digest, result)
+}
+
+pub fn generate_cdag(cache: &[Node]) -> CDag {
+	let mut c_dag = [0u32; PROGPOW_CACHE_WORDS];
+
+	for i in 0..PROGPOW_CACHE_WORDS / 16 {
+		let node = calculate_dag_item(i as u32, cache);
+		for j in 0..16 {
+			c_dag[i * 16 + j] = node.as_words()[j];
+		}
+	}
+
+	c_dag
+}
+
+#[cfg(test)]
+mod test {
+	use tempdir::TempDir;
+
+	use cache::{NodeCacheBuilder, OptimizeFor};
+	use keccak::H256;
+	use rustc_hex::FromHex;
+	use serde_json::{self, Value};
+	use std::collections::VecDeque;
+	use super::*;
+
+	fn h256(hex: &str) -> H256 {
+		let bytes = FromHex::from_hex(hex).unwrap();
+		let mut res = [0; 32];
+		res.copy_from_slice(&bytes);
+		res
+	}
+
+	#[test]
+	fn test_cdag() {
+		let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
+		let tempdir = TempDir::new("").unwrap();
+		let cache = builder.new_cache(tempdir.into_path(), 0);
+
+		let c_dag = generate_cdag(cache.as_ref());
+
+		let expected = vec![
+			690150178u32, 1181503948, 2248155602, 2118233073, 2193871115,
+			1791778428, 1067701239, 724807309, 530799275, 3480325829, 3899029234,
+			1998124059, 2541974622, 1100859971, 1297211151, 3268320000, 2217813733,
+			2690422980, 3172863319, 2651064309
+		];
+
+		assert_eq!(
+			c_dag.iter().take(20).cloned().collect::<Vec<_>>(),
+			expected,
+		);
+	}
+
+	#[test]
+	fn test_random_merge() {
+		let tests = [
+			(1000000u32, 101u32, 33000101u32),
+			(2000000, 102, 66003366),
+			(3000000, 103, 6000103),
+			(4000000, 104, 2000104),
+			(1000000, 0, 33000000),
+			(2000000, 0, 66000000),
+			(3000000, 0, 6000000),
+			(4000000, 0, 2000000),
+		];
+
+		for (i, &(a, b, expected)) in tests.iter().enumerate() {
+			assert_eq!(
+				merge(a, b, i as u32),
+				expected,
+			);
+		}
+	}
+
+	#[test]
+	fn test_random_math() {
+		let tests = [
+			(20u32, 22u32, 42u32),
+			(70000, 80000, 1305032704),
+			(70000, 80000, 1),
+			(1, 2, 1),
+			(3, 10000, 196608),
+			(3, 0, 3),
+			(3, 6, 2),
+			(3, 6, 7),
+			(3, 6, 5),
+			(0, 0xffffffff, 32),
+			(3 << 13, 1 << 5, 3),
+			(22, 20, 42),
+			(80000, 70000, 1305032704),
+			(80000, 70000, 1),
+			(2, 1, 1),
+			(10000, 3, 80000),
+			(0, 3, 0),
+			(6, 3, 2),
+			(6, 3, 7),
+			(6, 3, 5),
+			(0, 0xffffffff, 32),
+			(3 << 13, 1 << 5, 3),
+		];
+
+		for (i, &(a, b, expected)) in tests.iter().enumerate() {
+			assert_eq!(
+				math(a, b, i as u32),
+				expected,
+			);
+		}
+	}
+
+	#[test]
+	fn test_keccak_256() {
+		let expected = "5dd431e5fbc604f499bfa0232f45f8f142d0ff5178f539e5a7800bf0643697af";
+		assert_eq!(
+			keccak_f800_long([0; 32], 0, [0; 8]),
+			h256(expected),
+		);
+	}
+
+	#[test]
+	fn test_keccak_64() {
+		let expected: u64 = 0x5dd431e5fbc604f4;
+		assert_eq!(
+			keccak_f800_short([0; 32], 0, [0; 8]),
+			expected,
+		);
+	}
+
+	#[test]
+	fn test_progpow_hash() {
+		let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
+		let tempdir = TempDir::new("").unwrap();
+		let cache = builder.new_cache(tempdir.into_path(), 0);
+		let c_dag = generate_cdag(cache.as_ref());
+
+		let header_hash = [0; 32];
+
+		let (digest, result) = progpow(
+			header_hash,
+			0,
+			0,
+			cache.as_ref(),
+			&c_dag,
+		);
+
+		let expected_digest = FromHex::from_hex("63155f732f2bf556967f906155b510c917e48e99685ead76ea83f4eca03ab12b").unwrap();
+		let expected_result = FromHex::from_hex("faeb1be51075b03a4ff44b335067951ead07a3b078539ace76fd56fc410557a3").unwrap();
+
+		assert_eq!(
+			digest.to_vec(),
+			expected_digest,
+		);
+
+		assert_eq!(
+			result.to_vec(),
+			expected_result,
+		);
+	}
+
+	#[test]
+	fn test_progpow_testvectors() {
+		struct ProgpowTest {
+			block_number: u64,
+			header_hash: H256,
+			nonce: u64,
+			mix_hash: H256,
+			final_hash: H256,
+		}
+
+		let tests: Vec<VecDeque<Value>> =
+			serde_json::from_slice(include_bytes!("../res/progpow_testvectors.json")).unwrap();
+
+		let tests: Vec<ProgpowTest> = tests.into_iter().map(|mut test: VecDeque<Value>| {
+			assert!(test.len() == 5);
+
+			let block_number: u64 = serde_json::from_value(test.pop_front().unwrap()).unwrap();
+			let header_hash: String = serde_json::from_value(test.pop_front().unwrap()).unwrap();
+			let nonce: String = serde_json::from_value(test.pop_front().unwrap()).unwrap();
+			let mix_hash: String = serde_json::from_value(test.pop_front().unwrap()).unwrap();
+			let final_hash: String = serde_json::from_value(test.pop_front().unwrap()).unwrap();
+
+			ProgpowTest {
+				block_number,
+				header_hash: h256(&header_hash),
+				nonce: u64::from_str_radix(&nonce, 16).unwrap(),
+				mix_hash: h256(&mix_hash),
+				final_hash: h256(&final_hash),
+			}
+		}).collect();
+
+		for test in tests {
+			let builder = NodeCacheBuilder::new(OptimizeFor::Memory, u64::max_value());
+			let tempdir = TempDir::new("").unwrap();
+			let cache = builder.new_cache(tempdir.path().to_owned(), test.block_number);
+			let c_dag = generate_cdag(cache.as_ref());
+
+			let (digest, result) = progpow(
+				test.header_hash,
+				test.nonce,
+				test.block_number,
+				cache.as_ref(),
+				&c_dag,
+			);
+
+			assert_eq!(digest, test.final_hash);
+			assert_eq!(result, test.mix_hash);
+		}
+	}
+}

ethcore/Cargo.toml

@@ -107,7 +107,7 @@ slow-blocks = []
 # Run JSON consensus tests.
 json-tests = ["env_logger", "test-helpers", "to-pod-full"]
 # Skip JSON consensus tests with pending issues.
-ci-skip-issue = []
+ci-skip-tests = []
 # Run memory/cpu heavy tests.
 test-heavy = []
 # Compile test helpers

ethcore/src/ethereum/ethash.rs

@@ -113,6 +113,8 @@ pub struct EthashParams {
 	pub block_reward_contract: Option<BlockRewardContract>,
 	/// Difficulty bomb delays.
 	pub difficulty_bomb_delays: BTreeMap<BlockNumber, BlockNumber>,
+	/// Block to transition to progpow
+	pub progpow_transition: u64,
 }
 
 impl From<ethjson::spec::EthashParams> for EthashParams {
@@ -153,6 +155,7 @@ impl From<ethjson::spec::EthashParams> for EthashParams {
 				}),
 			expip2_transition: p.expip2_transition.map_or(u64::max_value(), Into::into),
 			expip2_duration_limit: p.expip2_duration_limit.map_or(30, Into::into),
+			progpow_transition: p.progpow_transition.map_or(u64::max_value(), Into::into),
 			block_reward_contract_transition: p.block_reward_contract_transition.map_or(0, Into::into),
 			block_reward_contract: match (p.block_reward_contract_code, p.block_reward_contract_address) {
 				(Some(code), _) => Some(BlockRewardContract::new_from_code(Arc::new(code.into()))),
@@ -182,10 +185,12 @@ impl Ethash {
 		machine: EthereumMachine,
 		optimize_for: T,
 	) -> Arc<Self> {
+		let progpow_transition = ethash_params.progpow_transition;
+
 		Arc::new(Ethash {
 			ethash_params,
 			machine,
-			pow: EthashManager::new(cache_dir.as_ref(), optimize_for.into()),
+			pow: EthashManager::new(cache_dir.as_ref(), optimize_for.into(), progpow_transition),
 		})
 	}
 }
@@ -320,7 +325,8 @@ impl Engine<EthereumMachine> for Arc<Ethash> {
 		let difficulty = ethash::boundary_to_difficulty(&H256(quick_get_difficulty(
 			&header.bare_hash().0,
 			seal.nonce.low_u64(),
-			&seal.mix_hash.0
+			&seal.mix_hash.0,
+			header.number() >= self.ethash_params.progpow_transition
 		)));
 
 		if &difficulty < header.difficulty() {
@@ -523,6 +529,7 @@ mod tests {
 			block_reward_contract: None,
 			block_reward_contract_transition: 0,
 			difficulty_bomb_delays: BTreeMap::new(),
+			progpow_transition: u64::max_value(),
 		}
 	}
 

ethcore/src/json_tests/skip.rs

@@ -18,10 +18,7 @@
 
 use ethjson;
 
-#[cfg(all(not(test), feature = "ci-skip-tests"))]
+#[cfg(feature="ci-skip-tests")]
-compile_error!("ci-skip-tests can only be enabled for testing builds.");
-
-#[cfg(feature="ci-skip-issue")]
 lazy_static!{
 	pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = {
 		let skip_data = include_bytes!("../../res/ethereum/tests-issues/currents.json");
@@ -29,7 +26,7 @@ lazy_static!{
 	};
 }
 
-#[cfg(not(feature="ci-skip-issue"))]
+#[cfg(not(feature="ci-skip-tests"))]
 lazy_static!{
 	pub static ref SKIP_TEST_STATE: ethjson::test::SkipStates = {
 		ethjson::test::SkipStates::empty()

json/src/spec/ethash.rs

@@ -95,6 +95,9 @@ pub struct EthashParams {
 	pub expip2_transition: Option<Uint>,
 	/// EXPIP-2 duration limit
 	pub expip2_duration_limit: Option<Uint>,
+	/// Block to transition to progpow
+	#[serde(rename="progpowTransition")]
+	pub progpow_transition: Option<Uint>,
 }
 
 /// Ethash engine deserialization.
@@ -200,6 +203,7 @@ mod tests {
 				ecip1017_era_rounds: None,
 				expip2_transition: None,
 				expip2_duration_limit: None,
+				progpow_transition: None,
 				difficulty_bomb_delays: None,
 			}
 		});
@@ -239,6 +243,7 @@ mod tests {
 				ecip1017_era_rounds: None,
 				expip2_transition: None,
 				expip2_duration_limit: None,
+				progpow_transition: None,
 				difficulty_bomb_delays: None,
 			}
 		});

test.sh

@@ -2,7 +2,7 @@
 # Running Parity Full Test Suite
 echo "________Running test.sh________"
 
-FEATURES="json-tests,ci-skip-issue"
+FEATURES="json-tests,ci-skip-tests"
 OPTIONS="--release"
 VALIDATE=1
 THREADS=8