PoA warp implementation (#5488)

* separate modules for consensus snapshot chunks

* bulk of authority warp logic

* finish authority warp implementation

* squash warnings and enable authority snapshot mode

* test harness for PoA

* fiddle with harness

* epoch generation proof fixes

* update constructor code

* store epoch transition proof after block commit

* basic snap and restore test

* use keyvaluedb in state restoration

* decompress chunks

* fix encoding issues

* fixed-to-contract-to-contract test

* implement ancient block import

* restore genesis transition in PoW snapshot

* add format version method to snapshot components

* supported version numbers in snapshot_components

* allow returning of ancient epoch transitions

* genesis hash mismatch check

* remove commented code

devtools/src/random_path.rs

@@ -96,7 +96,7 @@ impl Drop for RandomTempPath {
 
 pub struct GuardedTempResult<T> {
 	pub result: Option<T>,
-	pub _temp: RandomTempPath
+	pub _temp: RandomTempPath,
 }
 
 impl<T> GuardedTempResult<T> {

ethcore/Cargo.toml

@@ -52,6 +52,9 @@ stats = { path = "../util/stats" }
 time = "0.1"
 transient-hashmap = "0.4"
 
+[dev-dependencies]
+native-contracts = { path = "native_contracts", features = ["test_contracts"] }
+
 [features]
 jit = ["evmjit"]
 evm-debug = ["slow-blocks"]

ethcore/native_contracts/Cargo.toml

@@ -13,3 +13,6 @@ ethcore-util = { path = "../../util" }
 
 [build-dependencies]
 native-contract-generator = { path = "generator" }
+
+[features]
+default = []

ethcore/native_contracts/build.rs

@@ -33,6 +33,8 @@ const VALIDATOR_SET_ABI: &'static str = r#"[{"constant":true,"inputs":[],"name":
 
 const VALIDATOR_REPORT_ABI: &'static str = r#"[{"constant":false,"inputs":[{"name":"validator","type":"address"},{"name":"blockNumber","type":"uint256"},{"name":"proof","type":"bytes"}],"name":"reportMalicious","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"validator","type":"address"},{"name":"blockNumber","type":"uint256"}],"name":"reportBenign","outputs":[],"payable":false,"type":"function"}]"#;
 
+const TEST_VALIDATOR_SET_ABI: &'static str = r#"[{"constant":true,"inputs":[],"name":"transitionNonce","outputs":[{"name":"n","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"newValidators","type":"address[]"}],"name":"setValidators","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"getValidators","outputs":[{"name":"vals","type":"address[]"}],"payable":false,"type":"function"},{"inputs":[],"payable":false,"type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"_parent_hash","type":"bytes32"},{"indexed":true,"name":"_nonce","type":"uint256"},{"indexed":false,"name":"_new_set","type":"address[]"}],"name":"ValidatorsChanged","type":"event"}]"#;
+
 fn build_file(name: &str, abi: &str, filename: &str) {
 	let code = ::native_contract_generator::generate_module(name, abi).unwrap();
 
@@ -43,10 +45,16 @@ fn build_file(name: &str, abi: &str, filename: &str) {
 	f.write_all(code.as_bytes()).unwrap();
 }
 
+fn build_test_contracts() {
+	build_file("ValidatorSet", TEST_VALIDATOR_SET_ABI, "test_validator_set.rs");
+}
+
 fn main() {
 	build_file("Registry", REGISTRY_ABI, "registry.rs");
 	build_file("ServiceTransactionChecker", SERVICE_TRANSACTION_ABI, "service_transaction.rs");
 	build_file("SecretStoreAclStorage", SECRETSTORE_ACL_STORAGE_ABI, "secretstore_acl_storage.rs");
 	build_file("ValidatorSet", VALIDATOR_SET_ABI, "validator_set.rs");
 	build_file("ValidatorReport", VALIDATOR_REPORT_ABI, "validator_report.rs");
+
+	build_test_contracts();
 }

ethcore/native_contracts/src/lib.rs

@@ -29,6 +29,8 @@ mod secretstore_acl_storage;
 mod validator_set;
 mod validator_report;
 
+pub mod test_contracts;
+
 pub use self::registry::Registry;
 pub use self::service_transaction::ServiceTransactionChecker;
 pub use self::secretstore_acl_storage::SecretStoreAclStorage;

ethcore/native_contracts/src/test_contracts/mod.rs

@@ -0,0 +1,21 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Contracts used for testing.
+
+pub mod validator_set;
+
+pub use self::validator_set::ValidatorSet;

ethcore/native_contracts/src/test_contracts/validator_set.rs

@@ -0,0 +1,21 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+#![allow(unused_mut, unused_variables, unused_imports)]
+
+//! Test validator set contract.
+
+include!(concat!(env!("OUT_DIR"), "/test_validator_set.rs"));

ethcore/src/blockchain/blockchain.rs

@@ -445,7 +445,12 @@ impl<'a> Iterator for EpochTransitionIter<'a> {
 						let is_in_canon_chain = self.chain.block_hash(transition.block_number)
 							.map_or(false, |hash| hash == transition.block_hash);
 
-						if is_in_canon_chain {
+						// if the transition is within the block gap, there will only be
+						// one candidate, and it will be from a snapshot restored from.
+						let is_ancient = self.chain.first_block_number()
+							.map_or(false, |first| first > transition.block_number);
+
+						if is_ancient || is_in_canon_chain {
 							return Some((transitions.number, transition))
 						}
 					}
@@ -864,6 +869,7 @@ impl BlockChain {
 	}
 
 	/// Iterate over all epoch transitions.
+	/// This will only return transitions within the canonical chain.
 	pub fn epoch_transitions(&self) -> EpochTransitionIter {
 		let iter = self.db.iter_from_prefix(db::COL_EXTRA, &EPOCH_KEY_PREFIX[..]);
 		EpochTransitionIter {
@@ -872,6 +878,16 @@ impl BlockChain {
 		}
 	}
 
+	/// Get a specific epoch transition by epoch number and provided block hash.
+	pub fn epoch_transition(&self, epoch_num: u64, block_hash: H256) -> Option<EpochTransition> {
+		trace!(target: "blockchain", "Loading epoch {} transition at block {}",
+			epoch_num, block_hash);
+
+		self.db.read(db::COL_EXTRA, &epoch_num).and_then(|transitions: EpochTransitions| {
+			transitions.candidates.into_iter().find(|c| c.block_hash == block_hash)
+		})
+	}
+
 	/// Add a child to a given block. Assumes that the block hash is in
 	/// the chain and the child's parent is this block.
 	///

ethcore/src/client/ancient_import.rs

@@ -0,0 +1,68 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Helper for ancient block import.
+
+use std::sync::Arc;
+
+use engines::{Engine, EpochVerifier, EpochChange};
+use error::Error;
+use header::Header;
+
+use rand::Rng;
+use util::RwLock;
+
+// do "heavy" verification on ~1/50 blocks, randomly sampled.
+const HEAVY_VERIFY_RATE: f32 = 0.02;
+
+/// Ancient block verifier: import an ancient sequence of blocks in order from a starting
+/// epoch.
+pub struct AncientVerifier {
+	cur_verifier: RwLock<Box<EpochVerifier>>,
+	engine: Arc<Engine>,
+}
+
+impl AncientVerifier {
+	/// Create a new ancient block verifier with the given engine and initial verifier.
+	pub fn new(engine: Arc<Engine>, start_verifier: Box<EpochVerifier>) -> Self {
+		AncientVerifier {
+			cur_verifier: RwLock::new(start_verifier),
+			engine: engine,
+		}
+	}
+
+	/// Verify the next block header, randomly choosing whether to do heavy or light
+	/// verification. If the block is the end of an epoch, updates the epoch verifier.
+	pub fn verify<R: Rng, F: Fn(u64) -> Result<Box<EpochVerifier>, Error>>(
+		&self,
+		rng: &mut R,
+		header: &Header,
+		block: &[u8],
+		receipts: &[::receipt::Receipt],
+		load_verifier: F,
+	) -> Result<(), ::error::Error> {
+		match rng.gen::<f32>() <= HEAVY_VERIFY_RATE {
+			true => self.cur_verifier.read().verify_heavy(header)?,
+			false => self.cur_verifier.read().verify_light(header)?,
+		}
+
+		if let EpochChange::Yes(num) = self.engine.is_epoch_end(header, Some(block), Some(receipts)) {
+			*self.cur_verifier.write() = load_verifier(num)?;
+		}
+
+		Ok(())
+	}
+}

ethcore/src/client/client.rs

@@ -34,6 +34,7 @@ use basic_types::Seal;
 use block::*;
 use blockchain::{BlockChain, BlockProvider, EpochTransition, TreeRoute, ImportRoute};
 use blockchain::extras::TransactionAddress;
+use client::ancient_import::AncientVerifier;
 use client::Error as ClientError;
 use client::{
 	BlockId, TransactionId, UncleId, TraceId, ClientConfig, BlockChainClient,
@@ -61,7 +62,7 @@ use service::ClientIoMessage;
 use snapshot::{self, io as snapshot_io};
 use spec::Spec;
 use state_db::StateDB;
-use state::{self, State, CleanupMode};
+use state::{self, State};
 use trace;
 use trace::{TraceDB, ImportRequest as TraceImportRequest, LocalizedTrace, Database as TraceDatabase};
 use trace::FlatTransactionTraces;
@@ -152,6 +153,7 @@ pub struct Client {
 	factories: Factories,
 	history: u64,
 	rng: Mutex<OsRng>,
+	ancient_verifier: Mutex<Option<AncientVerifier>>,
 	on_user_defaults_change: Mutex<Option<Box<FnMut(Option<Mode>) + 'static + Send>>>,
 	registrar: Mutex<Option<Registry>>,
 	exit_handler: Mutex<Option<Box<Fn(bool, Option<String>) + 'static + Send>>>,
@@ -241,6 +243,7 @@ impl Client {
 			factories: factories,
 			history: history,
 			rng: Mutex::new(OsRng::new().map_err(::util::UtilError::StdIo)?),
+			ancient_verifier: Mutex::new(None),
 			on_user_defaults_change: Mutex::new(None),
 			registrar: Mutex::new(None),
 			exit_handler: Mutex::new(None),
@@ -256,7 +259,11 @@ impl Client {
 		// ensure genesis epoch proof in the DB.
 		{
 			let chain = client.chain.read();
-			client.generate_epoch_proof(&spec.genesis_header(), 0, &*chain);
+			let gh = spec.genesis_header();
+			if chain.epoch_transition(0, spec.genesis_header().hash()).is_none() {
+				trace!(target: "client", "No genesis transition found.");
+				client.generate_epoch_proof(&gh, 0, &*chain);
+			}
 		}
 
 		if let Some(reg_addr) = client.additional_params().get("registrar").and_then(|s| Address::from_str(s).ok()) {
@@ -540,25 +547,56 @@ impl Client {
 	fn import_old_block(&self, block_bytes: Bytes, receipts_bytes: Bytes) -> Result<H256, ::error::Error> {
 		let block = BlockView::new(&block_bytes);
 		let header = block.header();
+		let receipts = ::rlp::decode_list(&receipts_bytes);
 		let hash = header.hash();
 		let _import_lock = self.import_lock.lock();
+
 		{
 			let _timer = PerfTimer::new("import_old_block");
-			let mut rng = self.rng.lock();
 			let chain = self.chain.read();
+			let mut ancient_verifier = self.ancient_verifier.lock();
 
-			// verify block.
-			::snapshot::verify_old_block(
-				&mut *rng,
-				&header,
-				&*self.engine,
-				&*chain,
-				Some(&block_bytes),
-				false,
-			)?;
+			{
+				// closure for verifying a block.
+				let verify_with = |verifier: &AncientVerifier| -> Result<(), ::error::Error> {
+					// verify the block, passing a closure used to load an epoch verifier
+					// by number.
+					verifier.verify(
+						&mut *self.rng.lock(),
+						&header,
+						&block_bytes,
+						&receipts,
+						|epoch_num| chain.epoch_transition(epoch_num, hash)
+							.ok_or(BlockError::UnknownEpochTransition(epoch_num))
+							.map_err(Into::into)
+							.and_then(|t| self.engine.epoch_verifier(&header, &t.proof))
+					)
+				};
+
+				// initialize the ancient block verifier if we don't have one already.
+				match &mut *ancient_verifier {
+					&mut Some(ref verifier) => {
+						verify_with(verifier)?
+					}
+					x @ &mut None => {
+						// load most recent epoch.
+						trace!(target: "client", "Initializing ancient block restoration.");
+						let current_epoch_data = chain.epoch_transitions()
+							.take_while(|&(_, ref t)| t.block_number < header.number())
+							.last()
+							.map(|(_, t)| t.proof)
+							.expect("At least one epoch entry (genesis) always stored; qed");
+
+						let current_verifier = self.engine.epoch_verifier(&header, &current_epoch_data)?;
+						let current_verifier = AncientVerifier::new(self.engine.clone(), current_verifier);
+
+						verify_with(&current_verifier)?;
+						*x = Some(current_verifier);
+					}
+				}
+			}
 
 			// Commit results
-			let receipts = ::rlp::decode_list(&receipts_bytes);
 			let mut batch = DBTransaction::new();
 			chain.insert_unordered_block(&mut batch, &block_bytes, receipts, None, false, true);
 			// Final commit to the DB
@@ -590,7 +628,7 @@ impl Client {
 		let entering_new_epoch = {
 			use engines::EpochChange;
 			match self.engine.is_epoch_end(block.header(), Some(block_data), Some(&receipts)) {
-				EpochChange::Yes(e, _) => Some((block.header().clone(), e)),
+				EpochChange::Yes(e) => Some((block.header().clone(), e)),
 				EpochChange::No => None,
 				EpochChange::Unsure(_) => {
 					warn!(target: "client", "Detected invalid engine implementation.");
@@ -641,7 +679,8 @@ impl Client {
 
 		let mut batch = DBTransaction::new();
 		let hash = header.hash();
-		debug!(target: "client", "Generating validation proof for block {}", hash);
+		debug!(target: "client", "Generating validation proof for epoch {} at block {}",
+			epoch_number, hash);
 
 		// proof is two-part. state items read in lexicographical order,
 		// and the secondary "proof" part.
@@ -880,8 +919,8 @@ impl Client {
 		let start_hash = match at {
 			BlockId::Latest => {
 				let start_num = match db.earliest_era() {
-					Some(era) => ::std::cmp::max(era, best_block_number - history),
-					None => best_block_number - history,
+					Some(era) => ::std::cmp::max(era, best_block_number.saturating_sub(history)),
+					None => best_block_number.saturating_sub(history),
 				};
 
 				match self.block_hash(BlockId::Number(start_num)) {
@@ -992,16 +1031,9 @@ impl BlockChainClient for Client {
 		let mut state = self.state_at(block).ok_or(CallError::StatePruned)?;
 		let original_state = if analytics.state_diffing { Some(state.clone()) } else { None };
 
-		let sender = t.sender();
-		let balance = state.balance(&sender).map_err(|_| CallError::StateCorrupt)?;
-		let needed_balance = t.value + t.gas * t.gas_price;
-		if balance < needed_balance {
-			// give the sender a sufficient balance
-			state.add_balance(&sender, &(needed_balance - balance), CleanupMode::NoEmpty)
-				.map_err(|_| CallError::StateCorrupt)?;
-		}
 		let options = TransactOptions { tracing: analytics.transaction_tracing, vm_tracing: analytics.vm_tracing, check_nonce: false };
-		let mut ret = Executive::new(&mut state, &env_info, &*self.engine, &self.factories.vm).transact(t, options)?;
+		let mut ret = Executive::new(&mut state, &env_info, &*self.engine, &self.factories.vm)
+			.transact_virtual(t, options)?;
 
 		// TODO gav move this into Executive.
 		if let Some(original) = original_state {
@@ -1023,7 +1055,6 @@ impl BlockChainClient for Client {
 		// that's just a copy of the state.
 		let original_state = self.state_at(block).ok_or(CallError::StatePruned)?;
 		let sender = t.sender();
-		let balance = original_state.balance(&sender).map_err(ExecutionError::from)?;
 		let options = TransactOptions { tracing: true, vm_tracing: false, check_nonce: false };
 
 		let cond = |gas| {
@@ -1032,15 +1063,8 @@ impl BlockChainClient for Client {
 			let tx = tx.fake_sign(sender);
 
 			let mut state = original_state.clone();
-			let needed_balance = tx.value + tx.gas * tx.gas_price;
-			if balance < needed_balance {
-				// give the sender a sufficient balance
-				state.add_balance(&sender, &(needed_balance - balance), CleanupMode::NoEmpty)
-					.map_err(ExecutionError::from)?;
-			}
-
 			Ok(Executive::new(&mut state, &env_info, &*self.engine, &self.factories.vm)
-				.transact(&tx, options.clone())
+				.transact_virtual(&tx, options.clone())
 				.map(|r| r.exception.is_none())
 				.unwrap_or(false))
 		};

ethcore/src/client/mod.rs

@@ -16,6 +16,7 @@
 
 //! Blockchain database client.
 
+mod ancient_import;
 mod config;
 mod error;
 mod test_client;

ethcore/src/engines/authority_round.rs

@@ -452,6 +452,10 @@ impl Engine for AuthorityRound {
 	fn sign(&self, hash: H256) -> Result<Signature, Error> {
 		self.signer.sign(hash).map_err(Into::into)
 	}
+
+	fn snapshot_components(&self) -> Option<Box<::snapshot::SnapshotComponents>> {
+		Some(Box::new(::snapshot::PoaSnapshot))
+	}
 }
 
 #[cfg(test)]

ethcore/src/engines/basic_authority.rs

@@ -216,6 +216,10 @@ impl Engine for BasicAuthority {
 	fn sign(&self, hash: H256) -> Result<Signature, Error> {
 		self.signer.sign(hash).map_err(Into::into)
 	}
+
+	fn snapshot_components(&self) -> Option<Box<::snapshot::SnapshotComponents>> {
+		Some(Box::new(::snapshot::PoaSnapshot))
+	}
 }
 
 #[cfg(test)]

ethcore/src/engines/epoch_verifier.rs

@@ -22,7 +22,7 @@ use header::Header;
 /// Verifier for all blocks within an epoch with self-contained state.
 ///
 /// See docs on `Engine` relating to proving functions for more details.
-pub trait EpochVerifier: Sync {
+pub trait EpochVerifier: Send + Sync {
 	/// Get the epoch number.
 	fn epoch_number(&self) -> u64;
 

ethcore/src/engines/mod.rs

@@ -107,8 +107,8 @@ pub enum EpochChange {
 	Unsure(Unsure),
 	/// No epoch change.
 	No,
-	/// Validation proof required, and the new epoch number and expected proof.
-	Yes(u64, Bytes),
+	/// Validation proof required, and the new epoch number.
+	Yes(u64),
 }
 
 /// More data required to determine if an epoch change occurred at a given block.
@@ -227,6 +227,9 @@ pub trait Engine : Sync + Send {
 	/// For example, for PoA chains the proof will be a validator set,
 	/// and the corresponding `EpochVerifier` can be used to correctly validate
 	/// all blocks produced under that `ValidatorSet`
+	///
+	/// It must be possible to generate an epoch proof for any block in an epoch,
+	/// and it should always be equivalent to the proof of the transition block.
 	fn epoch_proof(&self, _header: &Header, _caller: &Call)
 		-> Result<Vec<u8>, Error>
 	{
@@ -234,6 +237,11 @@ pub trait Engine : Sync + Send {
 	}
 
 	/// Whether an epoch change occurred at the given header.
+	///
+	/// If the block or receipts are required, return `Unsure` and the function will be
+	/// called again with them.
+	/// Return `Yes` or `No` when the answer is definitively known.
+	///
 	/// Should not interact with state.
 	fn is_epoch_end(&self, _header: &Header, _block: Option<&[u8]>, _receipts: Option<&[Receipt]>)
 		-> EpochChange

ethcore/src/engines/validator_set/multi.rs

@@ -76,23 +76,45 @@ impl ValidatorSet for Multi {
 		-> EpochChange
 	{
 		let (set_block, set) = self.correct_set_by_number(header.number());
+		let (next_set_block, _) = self.correct_set_by_number(header.number() + 1);
+
+		// multi-set transitions require epoch changes.
+		if next_set_block != set_block {
+			return EpochChange::Yes(next_set_block);
+		}
 
 		match set.is_epoch_end(header, block, receipts) {
-			EpochChange::Yes(num, proof) => EpochChange::Yes(set_block + num, proof),
+			EpochChange::Yes(num) => EpochChange::Yes(set_block + num),
 			other => other,
 		}
 	}
 
 	fn epoch_proof(&self, header: &Header, caller: &Call) -> Result<Vec<u8>, String> {
-		self.correct_set_by_number(header.number()).1.epoch_proof(header, caller)
+		let (set_block, set) = self.correct_set_by_number(header.number());
+		let (next_set_block, next_set) = self.correct_set_by_number(header.number() + 1);
+
+		if next_set_block != set_block {
+			return next_set.epoch_proof(header, caller);
+		}
+
+		set.epoch_proof(header, caller)
 	}
 
 	fn epoch_set(&self, header: &Header, proof: &[u8]) -> Result<(u64, super::SimpleList), ::error::Error> {
 		// "multi" epoch is the inner set's epoch plus the transition block to that set.
 		// ensures epoch increases monotonically.
 		let (set_block, set) = self.correct_set_by_number(header.number());
-		let (inner_epoch, list) = set.epoch_set(header, proof)?;
-		Ok((set_block + inner_epoch, list))
+		let (next_set_block, next_set) = self.correct_set_by_number(header.number() + 1);
+
+		// this block kicks off a new validator set -- get the validator set
+		// starting there.
+		if next_set_block != set_block {
+			let (inner_epoch, list) = next_set.epoch_set(header, proof)?;
+			Ok((next_set_block + inner_epoch, list))
+		} else {
+			let (inner_epoch, list) = set.epoch_set(header, proof)?;
+			Ok((set_block + inner_epoch, list))
+		}
 	}
 
 	fn contains_with_caller(&self, bh: &H256, address: &Address, caller: &Call) -> bool {

ethcore/src/engines/validator_set/safe_contract.rs

@@ -182,10 +182,9 @@ impl ValidatorSet for ValidatorSafeContract {
 							);
 
 						match (nonce, validators) {
-							(Some(nonce), Some(validators)) => {
-								let proof = encode_proof(nonce, &validators);
+							(Some(nonce), Some(_)) => {
 								let new_epoch = nonce.low_u64();
-								::engines::EpochChange::Yes(new_epoch, proof)
+								::engines::EpochChange::Yes(new_epoch)
 							}
 							_ => {
 								debug!(target: "engine", "Successfully decoded log turned out to be bad.");

ethcore/src/error.rs

@@ -169,6 +169,8 @@ pub enum BlockError {
 	UnknownParent(H256),
 	/// Uncle parent given is unknown.
 	UnknownUncleParent(H256),
+	/// No transition to epoch number.
+	UnknownEpochTransition(u64),
 }
 
 impl fmt::Display for BlockError {
@@ -202,6 +204,7 @@ impl fmt::Display for BlockError {
 			RidiculousNumber(ref oob) => format!("Implausible block number. {}", oob),
 			UnknownParent(ref hash) => format!("Unknown parent: {}", hash),
 			UnknownUncleParent(ref hash) => format!("Unknown uncle parent: {}", hash),
+			UnknownEpochTransition(ref num) => format!("Unknown transition to epoch number: {}", num),
 		};
 
 		f.write_fmt(format_args!("Block error ({})", msg))

ethcore/src/executive.rs

@@ -129,6 +129,21 @@ impl<'a, B: 'a + StateBackend> Executive<'a, B> {
 		}
 	}
 
+	/// Execute a transaction in a "virtual" context.
+	/// This will ensure the caller has enough balance to execute the desired transaction.
+	/// Used for extra-block executions for things like consensus contracts and RPCs
+	pub fn transact_virtual(&'a mut self, t: &SignedTransaction, options: TransactOptions) -> Result<Executed, ExecutionError> {
+		let sender = t.sender();
+		let balance = self.state.balance(&sender)?;
+		let needed_balance = t.value + t.gas * t.gas_price;
+		if balance < needed_balance {
+			// give the sender a sufficient balance
+			self.state.add_balance(&sender, &(needed_balance - balance), CleanupMode::NoEmpty)?;
+		}
+
+		self.transact(t, options)
+	}
+
 	/// Execute transaction/call with tracing enabled
 	pub fn transact_with_tracer<T, V>(
 		&'a mut self,

ethcore/src/snapshot/consensus/authority.rs

@@ -0,0 +1,498 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Secondary chunk creation and restoration, implementation for proof-of-authority
+//! based engines.
+//!
+//! The chunks here contain state proofs of transitions, along with validator proofs.
+
+use super::{SnapshotComponents, Rebuilder, ChunkSink};
+
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+use blockchain::{BlockChain, BlockProvider, EpochTransition};
+use engines::{Engine, EpochVerifier};
+use env_info::EnvInfo;
+use ids::BlockId;
+use header::Header;
+use receipt::Receipt;
+use snapshot::{Error, ManifestData};
+use state_db::StateDB;
+
+use itertools::{Position, Itertools};
+use rlp::{RlpStream, UntrustedRlp};
+use util::{Address, Bytes, H256, KeyValueDB, DBValue};
+
+/// Snapshot creation and restoration for PoA chains.
+/// Chunk format:
+///
+/// [FLAG, [header, epoch_number, epoch data, state proof, last hashes], ...]
+///   - Header data at which transition occurred,
+///   - epoch data (usually list of validators)
+///   - state items required to check epoch data
+///   - last 256 hashes before the transition; required for checking state changes.
+///
+/// FLAG is a bool: true for last chunk, false otherwise.
+///
+/// The last item of the last chunk will be a list containing data for the warp target block:
+/// [header, transactions, uncles, receipts, last_hashes, parent_td].
+/// If this block is not a transition block, the epoch data should be the same as that
+/// for the last transition.
+pub struct PoaSnapshot;
+
+impl SnapshotComponents for PoaSnapshot {
+	fn chunk_all(
+		&mut self,
+		chain: &BlockChain,
+		block_at: H256,
+		sink: &mut ChunkSink,
+		preferred_size: usize,
+	) -> Result<(), Error> {
+		let number = chain.block_number(&block_at)
+			.ok_or_else(|| Error::InvalidStartingBlock(BlockId::Hash(block_at)))?;
+
+		let mut pending_size = 0;
+		let mut rlps = Vec::new();
+
+		// TODO: this will become irrelevant after recent block hashes are moved into
+		// the state. can we optimize it out in that case?
+		let make_last_hashes = |parent_hash| chain.ancestry_iter(parent_hash)
+			.into_iter()
+			.flat_map(|inner| inner)
+			.take(255)
+			.collect::<Vec<_>>();
+
+		for (epoch_number, transition) in chain.epoch_transitions()
+			.take_while(|&(_, ref t)| t.block_number <= number)
+		{
+			let header = chain.block_header_data(&transition.block_hash)
+				.ok_or(Error::BlockNotFound(transition.block_hash))?;
+
+			let last_hashes: Vec<_> = make_last_hashes(header.parent_hash());
+
+			let entry = {
+				let mut entry_stream = RlpStream::new_list(5);
+				entry_stream
+					.append_raw(&header.into_inner(), 1)
+					.append(&epoch_number)
+					.append(&transition.proof);
+
+				entry_stream.begin_list(transition.state_proof.len());
+				for item in transition.state_proof {
+					entry_stream.append(&&*item);
+				}
+
+				entry_stream.append_list(&last_hashes);
+				entry_stream.out()
+			};
+
+			// cut of the chunk if too large.
+			let new_loaded_size = pending_size + entry.len();
+			pending_size = if new_loaded_size > preferred_size && !rlps.is_empty() {
+				write_chunk(false, &mut rlps, sink)?;
+				entry.len()
+			} else {
+				new_loaded_size
+			};
+
+			rlps.push(entry);
+		}
+
+		let (block, receipts) = chain.block(&block_at)
+			.and_then(|b| chain.block_receipts(&block_at).map(|r| (b, r)))
+			.ok_or(Error::BlockNotFound(block_at))?;
+		let block = block.decode();
+
+		let parent_td = chain.block_details(block.header.parent_hash())
+			.map(|d| d.total_difficulty)
+			.ok_or(Error::BlockNotFound(block_at))?;
+
+		let last_hashes = make_last_hashes(*block.header.parent_hash());
+
+		rlps.push({
+			let mut stream = RlpStream::new_list(6);
+			stream
+				.append(&block.header)
+				.append_list(&block.transactions)
+				.append_list(&block.uncles)
+				.append(&receipts)
+				.append_list(&last_hashes)
+				.append(&parent_td);
+			stream.out()
+		});
+
+		write_chunk(true, &mut rlps, sink)?;
+
+		Ok(())
+	}
+
+	fn rebuilder(
+		&self,
+		chain: BlockChain,
+		db: Arc<KeyValueDB>,
+		manifest: &ManifestData,
+	) -> Result<Box<Rebuilder>, ::error::Error> {
+		Ok(Box::new(ChunkRebuilder {
+			manifest: manifest.clone(),
+			warp_target: None,
+			chain: chain,
+			db: db,
+			had_genesis: false,
+			unverified_firsts: Vec::new(),
+			last_proofs: Vec::new(),
+		}))
+	}
+
+	fn min_supported_version(&self) -> u64 { 3 }
+	fn current_version(&self) -> u64 { 3 }
+}
+
+// writes a chunk composed of the inner RLPs here.
+// flag indicates whether the chunk is the last chunk.
+fn write_chunk(last: bool, chunk_data: &mut Vec<Bytes>, sink: &mut ChunkSink) -> Result<(), Error> {
+	let mut stream = RlpStream::new_list(1 + chunk_data.len());
+
+	stream.append(&last);
+	for item in chunk_data.drain(..) {
+		stream.append_raw(&item, 1);
+	}
+
+	(sink)(stream.out().as_slice()).map_err(Into::into)
+}
+
+// rebuilder checks state proofs for all transitions, and checks that each
+// transition header is verifiable from the epoch data of the one prior.
+struct ChunkRebuilder {
+	manifest: ManifestData,
+	warp_target: Option<(Header, Vec<H256>)>,
+	chain: BlockChain,
+	db: Arc<KeyValueDB>,
+	had_genesis: bool,
+
+	// sorted vectors of unverified first blocks in a chunk
+	// and epoch data from last blocks in chunks.
+	// verification for these will be done at the end.
+	unverified_firsts: Vec<(u64, Header)>,
+	last_proofs: Vec<(u64, Header, Bytes)>,
+}
+
+// verified data.
+struct Verified {
+	epoch_number: u64,
+	epoch_transition: EpochTransition,
+	header: Header,
+}
+
+// make a transaction and env info.
+// TODO: hardcoded 50M to match constants in client.
+// would be nice to extract magic numbers, or better yet
+// off-chain transaction execution, into its own module.
+fn make_tx_and_env(
+	engine: &Engine,
+	addr: Address,
+	data: Bytes,
+	header: &Header,
+	last_hashes: Arc<Vec<H256>>,
+) -> (::transaction::SignedTransaction, EnvInfo) {
+	use transaction::{Action, Transaction};
+
+	let transaction = Transaction {
+		nonce: engine.account_start_nonce(),
+		action: Action::Call(addr),
+		gas: 50_000_000.into(),
+		gas_price: 0.into(),
+		value: 0.into(),
+		data: data,
+	}.fake_sign(Default::default());
+
+	let env = EnvInfo {
+		number: header.number(),
+		author: *header.author(),
+		timestamp: header.timestamp(),
+		difficulty: *header.difficulty(),
+		gas_limit: 50_000_000.into(),
+		last_hashes: last_hashes,
+		gas_used: 0.into(),
+	};
+
+	(transaction, env)
+}
+
+impl ChunkRebuilder {
+	fn verify_transition(
+		&mut self,
+		last_verifier: &mut Option<Box<EpochVerifier>>,
+		transition_rlp: UntrustedRlp,
+		engine: &Engine,
+	) -> Result<Verified, ::error::Error> {
+		// decode.
+		let header: Header = transition_rlp.val_at(0)?;
+		let epoch_number: u64 = transition_rlp.val_at(1)?;
+		let epoch_data: Bytes = transition_rlp.val_at(2)?;
+		let state_proof: Vec<DBValue> = transition_rlp.at(3)?
+			.iter()
+			.map(|x| Ok(DBValue::from_slice(x.data()?)))
+			.collect::<Result<_, ::rlp::DecoderError>>()?;
+		let last_hashes: Vec<H256> = transition_rlp.list_at(4)?;
+		let last_hashes = Arc::new(last_hashes);
+
+		trace!(target: "snapshot", "verifying transition to epoch {}", epoch_number);
+
+		// check current transition against validators of last epoch.
+		if let Some(verifier) = last_verifier.as_ref() {
+			verifier.verify_heavy(&header)?;
+		}
+
+		{
+			// check the provided state proof actually leads to the
+			// given epoch data.
+			let caller = |addr, data| {
+				use state::{check_proof, ProvedExecution};
+
+				let (transaction, env_info) = make_tx_and_env(
+					engine,
+					addr,
+					data,
+					&header,
+					last_hashes.clone(),
+				);
+
+				let result = check_proof(
+					&state_proof,
+					header.state_root().clone(),
+					&transaction,
+					engine,
+					&env_info,
+				);
+
+				match result {
+					ProvedExecution::Complete(executed) => Ok(executed.output),
+					_ => Err("Bad state proof".into()),
+				}
+			};
+
+			let extracted_proof = engine.epoch_proof(&header, &caller)
+				.map_err(|_| Error::BadEpochProof(epoch_number))?;
+
+			if extracted_proof != epoch_data {
+				return Err(Error::BadEpochProof(epoch_number).into());
+			}
+		}
+
+		// create new epoch verifier.
+		*last_verifier = Some(engine.epoch_verifier(&header, &epoch_data)?);
+
+		Ok(Verified {
+			epoch_number: epoch_number,
+			epoch_transition: EpochTransition {
+				block_hash: header.hash(),
+				block_number: header.number(),
+				state_proof: state_proof,
+				proof: epoch_data,
+			},
+			header: header,
+		})
+	}
+}
+
+impl Rebuilder for ChunkRebuilder {
+	fn feed(
+		&mut self,
+		chunk: &[u8],
+		engine: &Engine,
+		abort_flag: &AtomicBool,
+	) -> Result<(), ::error::Error> {
+		let rlp = UntrustedRlp::new(chunk);
+		let is_last_chunk: bool = rlp.val_at(0)?;
+		let num_items = rlp.item_count()?;
+
+		// number of transitions in the chunk.
+		let num_transitions = if is_last_chunk {
+			num_items - 2
+		} else {
+			num_items - 1
+		};
+
+		let mut last_verifier = None;
+		let mut last_number = None;
+		for transition_rlp in rlp.iter().skip(1).take(num_transitions).with_position() {
+			if !abort_flag.load(Ordering::SeqCst) { return Err(Error::RestorationAborted.into()) }
+
+			let (is_first, is_last) = match transition_rlp {
+				Position::First(_) => (true, false),
+				Position::Middle(_) => (false, false),
+				Position::Last(_) => (false, true),
+				Position::Only(_) => (true, true),
+			};
+
+			let transition_rlp = transition_rlp.into_inner();
+			let verified = self.verify_transition(
+				&mut last_verifier,
+				transition_rlp,
+				engine,
+			)?;
+
+			if last_number.map_or(false, |num| verified.header.number() <= num) {
+				return Err(Error::WrongChunkFormat("Later epoch transition in earlier or same block.".into()).into());
+			}
+
+			last_number = Some(verified.header.number());
+
+			// book-keep borders for verification later.
+			if is_first {
+				// make sure the genesis transition was included,
+				// but it doesn't need verification later.
+				if verified.epoch_number == 0 && verified.header.number() == 0 {
+					if verified.header.hash() != self.chain.genesis_hash() {
+						return Err(Error::WrongBlockHash(0, verified.header.hash(), self.chain.genesis_hash()).into());
+					}
+
+					self.had_genesis = true;
+				} else {
+					let idx = self.unverified_firsts
+						.binary_search_by_key(&verified.epoch_number, |&(a, _)| a)
+						.unwrap_or_else(|x| x);
+
+					let entry = (verified.epoch_number, verified.header.clone());
+					self.unverified_firsts.insert(idx, entry);
+				}
+			}
+			if is_last {
+				let idx = self.last_proofs
+					.binary_search_by_key(&verified.epoch_number, |&(a, _, _)| a)
+					.unwrap_or_else(|x| x);
+
+				let entry = (
+					verified.epoch_number,
+					verified.header.clone(),
+					verified.epoch_transition.proof.clone()
+				);
+				self.last_proofs.insert(idx, entry);
+			}
+
+			// write epoch transition into database.
+			let mut batch = self.db.transaction();
+			self.chain.insert_epoch_transition(&mut batch, verified.epoch_number,
+				verified.epoch_transition);
+			self.db.write_buffered(batch);
+
+			trace!(target: "snapshot", "Verified epoch transition for epoch {}", verified.epoch_number);
+		}
+
+		if is_last_chunk {
+			use block::Block;
+
+			let last_rlp = rlp.at(num_items - 1)?;
+			let block = Block {
+				header: last_rlp.val_at(0)?,
+				transactions: last_rlp.list_at(1)?,
+				uncles: last_rlp.list_at(2)?,
+			};
+			let block_data = block.rlp_bytes(::basic_types::Seal::With);
+			let receipts: Vec<Receipt> = last_rlp.list_at(3)?;
+
+			{
+				let hash = block.header.hash();
+				let best_hash = self.manifest.block_hash;
+				if hash != best_hash {
+					return Err(Error::WrongBlockHash(block.header.number(), best_hash, hash).into())
+				}
+			}
+
+			let last_hashes: Vec<H256> = last_rlp.list_at(4)?;
+			let parent_td: ::util::U256 = last_rlp.val_at(5)?;
+
+			let mut batch = self.db.transaction();
+			self.chain.insert_unordered_block(&mut batch, &block_data, receipts, Some(parent_td), true, false);
+			self.db.write_buffered(batch);
+
+			self.warp_target = Some((block.header, last_hashes));
+		}
+
+		Ok(())
+	}
+
+	fn finalize(&mut self, db: StateDB, engine: &Engine) -> Result<(), ::error::Error> {
+		use state::State;
+
+		if !self.had_genesis {
+			return Err(Error::WrongChunkFormat("No genesis transition included.".into()).into());
+		}
+
+		let (target_header, target_last_hashes) = match self.warp_target.take() {
+			Some(x) => x,
+			None => return Err(Error::WrongChunkFormat("Warp target block not included.".into()).into()),
+		};
+
+		// we store the last data even for the last chunk for easier verification
+		// of warp target, but we don't store genesis transition data.
+		// other than that, there should be a one-to-one correspondence of
+		// chunk ends to chunk beginnings.
+		if self.last_proofs.len() != self.unverified_firsts.len() + 1 {
+			return Err(Error::WrongChunkFormat("More than one 'last' chunk".into()).into());
+		}
+
+		// verify the first entries of chunks we couldn't before.
+		let lasts_iter = self.last_proofs.iter().map(|&(_, ref hdr, ref proof)| (hdr, &proof[..]));
+		let firsts_iter = self.unverified_firsts.iter().map(|&(_, ref hdr)| hdr);
+
+		for ((last_hdr, last_proof), first_hdr) in lasts_iter.zip(firsts_iter) {
+			let verifier = engine.epoch_verifier(&last_hdr, &last_proof)?;
+			verifier.verify_heavy(&first_hdr)?;
+		}
+
+		// verify that the validator set of the warp target is the same as that of the
+		// most recent transition. if the warp target was a transition itself,
+		// `last_data` will still be correct
+		let &(_, _, ref last_data) = self.last_proofs.last()
+			.expect("last_proofs known to have at least one element by the check above; qed");
+
+		let target_last_hashes = Arc::new(target_last_hashes);
+		let caller = |addr, data| {
+			use executive::{Executive, TransactOptions};
+
+			let factories = ::factory::Factories::default();
+			let mut state = State::from_existing(
+				db.boxed_clone(),
+				self.manifest.state_root.clone(),
+				engine.account_start_nonce(),
+				factories.clone(),
+			).map_err(|e| format!("State root mismatch: {}", e))?;
+
+			let (tx, env_info) = make_tx_and_env(
+				engine,
+				addr,
+				data,
+				&target_header,
+				target_last_hashes.clone(),
+			);
+
+			let options = TransactOptions { tracing: false, vm_tracing: false, check_nonce: false };
+			Executive::new(&mut state, &env_info, engine, &factories.vm)
+				.transact_virtual(&tx, options)
+				.map(|e| e.output)
+				.map_err(|e| format!("Error executing: {}", e))
+		};
+
+		let data = engine.epoch_proof(&target_header, &caller)?;
+		if &data[..] != &last_data[..] {
+			return Err(Error::WrongChunkFormat("Warp target has different epoch data than epoch transition.".into()).into())
+		}
+
+		Ok(())
+	}
+}

ethcore/src/snapshot/consensus/mod.rs

@@ -17,24 +17,24 @@
 //! Secondary chunk creation and restoration, implementations for different consensus
 //! engines.
 
-use std::collections::VecDeque;
-use std::io;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::AtomicBool;
 use std::sync::Arc;
 
-use blockchain::{BlockChain, BlockProvider};
+use blockchain::BlockChain;
 use engines::Engine;
 use snapshot::{Error, ManifestData};
-use snapshot::block::AbridgedBlock;
 
-use util::{Bytes, H256};
+use util::H256;
 use util::kvdb::KeyValueDB;
-use rand::OsRng;
-use rlp::{RlpStream, UntrustedRlp};
 
+mod authority;
+mod work;
+
+pub use self::authority::*;
+pub use self::work::*;
 
 /// A sink for produced chunks.
-pub type ChunkSink<'a> = FnMut(&[u8]) -> io::Result<()> + 'a;
+pub type ChunkSink<'a> = FnMut(&[u8]) -> ::std::io::Result<()> + 'a;
 
 /// Components necessary for snapshot creation and restoration.
 pub trait SnapshotComponents: Send {
@@ -57,13 +57,21 @@ pub trait SnapshotComponents: Send {
 	/// order and then be finalized.
 	///
 	/// The manifest, a database, and fresh `BlockChain` are supplied.
-	// TODO: supply anything for state?
+	///
+	/// The engine passed to the `Rebuilder` methods will be the same instance
+	/// that created the `SnapshotComponents`.
 	fn rebuilder(
 		&self,
 		chain: BlockChain,
 		db: Arc<KeyValueDB>,
 		manifest: &ManifestData,
 	) -> Result<Box<Rebuilder>, ::error::Error>;
+
+	/// Minimum supported snapshot version number.
+	fn min_supported_version(&self) -> u64;
+
+	/// Current version number
+	fn current_version(&self) -> u64;
 }
 
 
@@ -82,271 +90,10 @@ pub trait Rebuilder: Send {
 
 	/// Finalize the restoration. Will be done after all chunks have been
 	/// fed successfully.
-	/// This will apply the necessary "glue" between chunks.
-	fn finalize(&mut self) -> Result<(), Error>;
-}
-
-/// Snapshot creation and restoration for PoW chains.
-/// This includes blocks from the head of the chain as a
-/// loose assurance that the chain is valid.
-///
-/// The field is the number of blocks from the head of the chain
-/// to include in the snapshot.
-#[derive(Clone, Copy, PartialEq)]
-pub struct PowSnapshot(pub u64);
-
-impl SnapshotComponents for PowSnapshot {
-	fn chunk_all(
-		&mut self,
-		chain: &BlockChain,
-		block_at: H256,
-		chunk_sink: &mut ChunkSink,
-		preferred_size: usize,
-	) -> Result<(), Error> {
-		PowWorker {
-			chain: chain,
-			rlps: VecDeque::new(),
-			current_hash: block_at,
-			writer: chunk_sink,
-			preferred_size: preferred_size,
-		}.chunk_all(self.0)
-	}
-
-	fn rebuilder(
-		&self,
-		chain: BlockChain,
-		db: Arc<KeyValueDB>,
-		manifest: &ManifestData,
-	) -> Result<Box<Rebuilder>, ::error::Error> {
-		PowRebuilder::new(chain, db, manifest, self.0).map(|r| Box::new(r) as Box<_>)
-	}
-}
-
-/// Used to build block chunks.
-struct PowWorker<'a> {
-	chain: &'a BlockChain,
-	// block, receipt rlp pairs.
-	rlps: VecDeque<Bytes>,
-	current_hash: H256,
-	writer: &'a mut ChunkSink<'a>,
-	preferred_size: usize,
-}
-
-impl<'a> PowWorker<'a> {
-	// Repeatedly fill the buffers and writes out chunks, moving backwards from starting block hash.
-	// Loops until we reach the first desired block, and writes out the remainder.
-	fn chunk_all(&mut self, snapshot_blocks: u64) -> Result<(), Error> {
-		let mut loaded_size = 0;
-		let mut last = self.current_hash;
-
-		let genesis_hash = self.chain.genesis_hash();
-
-		for _ in 0..snapshot_blocks {
-			if self.current_hash == genesis_hash { break }
-
-			let (block, receipts) = self.chain.block(&self.current_hash)
-				.and_then(|b| self.chain.block_receipts(&self.current_hash).map(|r| (b, r)))
-				.ok_or(Error::BlockNotFound(self.current_hash))?;
-
-			let abridged_rlp = AbridgedBlock::from_block_view(&block.view()).into_inner();
-
-			let pair = {
-				let mut pair_stream = RlpStream::new_list(2);
-				pair_stream.append_raw(&abridged_rlp, 1).append(&receipts);
-				pair_stream.out()
-			};
-
-			let new_loaded_size = loaded_size + pair.len();
-
-			// cut off the chunk if too large.
-
-			if new_loaded_size > self.preferred_size && !self.rlps.is_empty() {
-				self.write_chunk(last)?;
-				loaded_size = pair.len();
-			} else {
-				loaded_size = new_loaded_size;
-			}
-
-			self.rlps.push_front(pair);
-
-			last = self.current_hash;
-			self.current_hash = block.header_view().parent_hash();
-		}
-
-		if loaded_size != 0 {
-			self.write_chunk(last)?;
-		}
-
-		Ok(())
-	}
-
-	// write out the data in the buffers to a chunk on disk
-	//
-	// we preface each chunk with the parent of the first block's details,
-	// obtained from the details of the last block written.
-	fn write_chunk(&mut self, last: H256) -> Result<(), Error> {
-		trace!(target: "snapshot", "prepared block chunk with {} blocks", self.rlps.len());
-
-		let (last_header, last_details) = self.chain.block_header(&last)
-			.and_then(|n| self.chain.block_details(&last).map(|d| (n, d)))
-			.ok_or(Error::BlockNotFound(last))?;
-
-		let parent_number = last_header.number() - 1;
-		let parent_hash = last_header.parent_hash();
-		let parent_total_difficulty = last_details.total_difficulty - *last_header.difficulty();
-
-		trace!(target: "snapshot", "parent last written block: {}", parent_hash);
-
-		let num_entries = self.rlps.len();
-		let mut rlp_stream = RlpStream::new_list(3 + num_entries);
-		rlp_stream.append(&parent_number).append(parent_hash).append(&parent_total_difficulty);
-
-		for pair in self.rlps.drain(..) {
-			rlp_stream.append_raw(&pair, 1);
-		}
-
-		let raw_data = rlp_stream.out();
-
-		(self.writer)(&raw_data)?;
-
-		Ok(())
-	}
-}
-
-/// Rebuilder for proof-of-work chains.
-/// Does basic verification for all blocks, but `PoW` verification for some.
-/// Blocks must be fed in-order.
-///
-/// The first block in every chunk is disconnected from the last block in the
-/// chunk before it, as chunks may be submitted out-of-order.
-///
-/// After all chunks have been submitted, we "glue" the chunks together.
-pub struct PowRebuilder {
-	chain: BlockChain,
-	db: Arc<KeyValueDB>,
-	rng: OsRng,
-	disconnected: Vec<(u64, H256)>,
-	best_number: u64,
-	best_hash: H256,
-	best_root: H256,
-	fed_blocks: u64,
-	snapshot_blocks: u64,
-}
-
-impl PowRebuilder {
-	/// Create a new PowRebuilder.
-	fn new(chain: BlockChain, db: Arc<KeyValueDB>, manifest: &ManifestData, snapshot_blocks: u64) -> Result<Self, ::error::Error> {
-		Ok(PowRebuilder {
-			chain: chain,
-			db: db,
-			rng: OsRng::new()?,
-			disconnected: Vec::new(),
-			best_number: manifest.block_number,
-			best_hash: manifest.block_hash,
-			best_root: manifest.state_root,
-			fed_blocks: 0,
-			snapshot_blocks: snapshot_blocks,
-		})
-	}
-}
-
-impl Rebuilder for PowRebuilder {
-	/// Feed the rebuilder an uncompressed block chunk.
-	/// Returns the number of blocks fed or any errors.
-	fn feed(&mut self, chunk: &[u8], engine: &Engine, abort_flag: &AtomicBool) -> Result<(), ::error::Error> {
-		use basic_types::Seal::With;
-		use views::BlockView;
-		use snapshot::verify_old_block;
-		use util::U256;
-		use util::triehash::ordered_trie_root;
-
-		let rlp = UntrustedRlp::new(chunk);
-		let item_count = rlp.item_count()?;
-		let num_blocks = (item_count - 3) as u64;
-
-		trace!(target: "snapshot", "restoring block chunk with {} blocks.", item_count - 3);
-
-		if self.fed_blocks + num_blocks > self.snapshot_blocks {
-			return Err(Error::TooManyBlocks(self.snapshot_blocks, self.fed_blocks).into())
-		}
-
-		// todo: assert here that these values are consistent with chunks being in order.
-		let mut cur_number = rlp.val_at::<u64>(0)? + 1;
-		let mut parent_hash = rlp.val_at::<H256>(1)?;
-		let parent_total_difficulty = rlp.val_at::<U256>(2)?;
-
-		for idx in 3..item_count {
-			if !abort_flag.load(Ordering::SeqCst) { return Err(Error::RestorationAborted.into()) }
-
-			let pair = rlp.at(idx)?;
-			let abridged_rlp = pair.at(0)?.as_raw().to_owned();
-			let abridged_block = AbridgedBlock::from_raw(abridged_rlp);
-			let receipts: Vec<::receipt::Receipt> = pair.list_at(1)?;
-			let receipts_root = ordered_trie_root(
-				pair.at(1)?.iter().map(|r| r.as_raw().to_owned())
-			);
-
-			let block = abridged_block.to_block(parent_hash, cur_number, receipts_root)?;
-			let block_bytes = block.rlp_bytes(With);
-			let is_best = cur_number == self.best_number;
-
-			if is_best {
-				if block.header.hash() != self.best_hash {
-					return Err(Error::WrongBlockHash(cur_number, self.best_hash, block.header.hash()).into())
-				}
-
-				if block.header.state_root() != &self.best_root {
-					return Err(Error::WrongStateRoot(self.best_root, *block.header.state_root()).into())
-				}
-			}
-
-			verify_old_block(
-				&mut self.rng,
-				&block.header,
-				engine,
-				&self.chain,
-				Some(&block_bytes),
-				is_best
-			)?;
-
-			let mut batch = self.db.transaction();
-
-			// special-case the first block in each chunk.
-			if idx == 3 {
-				if self.chain.insert_unordered_block(&mut batch, &block_bytes, receipts, Some(parent_total_difficulty), is_best, false) {
-					self.disconnected.push((cur_number, block.header.hash()));
-				}
-			} else {
-				self.chain.insert_unordered_block(&mut batch, &block_bytes, receipts, None, is_best, false);
-			}
-			self.db.write_buffered(batch);
-			self.chain.commit();
-
-			parent_hash = BlockView::new(&block_bytes).hash();
-			cur_number += 1;
-		}
-
-		self.fed_blocks += num_blocks;
-
-		Ok(())
-	}
-
-	/// Glue together any disconnected chunks and check that the chain is complete.
-	fn finalize(&mut self) -> Result<(), Error> {
-		let mut batch = self.db.transaction();
-
-		for (first_num, first_hash) in self.disconnected.drain(..) {
-			let parent_num = first_num - 1;
-
-			// check if the parent is even in the chain.
-			// since we don't restore every single block in the chain,
-			// the first block of the first chunks has nothing to connect to.
-			if let Some(parent_hash) = self.chain.block_hash(parent_num) {
-				// if so, add the child to it.
-				self.chain.add_child(&mut batch, parent_hash, first_hash);
-			}
-		}
-		self.db.write_buffered(batch);
-		Ok(())
-	}
+	///
+	/// This should apply the necessary "glue" between chunks,
+	/// and verify against the restored state.
+	///
+	/// The database passed contains the state for the warp target block.
+	fn finalize(&mut self, db: ::state_db::StateDB, engine: &Engine) -> Result<(), ::error::Error>;
 }

ethcore/src/snapshot/consensus/work.rs

@@ -0,0 +1,311 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Secondary chunk creation and restoration, implementation for proof-of-work
+//! chains.
+//!
+//! The secondary chunks in this instance are 30,000 "abridged blocks" from the head
+//! of the chain, which serve as an indication of valid chain.
+
+use super::{SnapshotComponents, Rebuilder, ChunkSink};
+
+use std::collections::VecDeque;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+use blockchain::{BlockChain, BlockProvider};
+use engines::Engine;
+use snapshot::{Error, ManifestData};
+use snapshot::block::AbridgedBlock;
+use util::{Bytes, H256, KeyValueDB};
+use rlp::{RlpStream, UntrustedRlp};
+use rand::OsRng;
+
+/// Snapshot creation and restoration for PoW chains.
+/// This includes blocks from the head of the chain as a
+/// loose assurance that the chain is valid.
+///
+/// The field is the number of blocks from the head of the chain
+/// to include in the snapshot.
+#[derive(Clone, Copy, PartialEq)]
+pub struct PowSnapshot(pub u64);
+
+impl SnapshotComponents for PowSnapshot {
+	fn chunk_all(
+		&mut self,
+		chain: &BlockChain,
+		block_at: H256,
+		chunk_sink: &mut ChunkSink,
+		preferred_size: usize,
+	) -> Result<(), Error> {
+		PowWorker {
+			chain: chain,
+			rlps: VecDeque::new(),
+			current_hash: block_at,
+			writer: chunk_sink,
+			preferred_size: preferred_size,
+		}.chunk_all(self.0)
+	}
+
+	fn rebuilder(
+		&self,
+		chain: BlockChain,
+		db: Arc<KeyValueDB>,
+		manifest: &ManifestData,
+	) -> Result<Box<Rebuilder>, ::error::Error> {
+		PowRebuilder::new(chain, db, manifest, self.0).map(|r| Box::new(r) as Box<_>)
+	}
+
+	fn min_supported_version(&self) -> u64 { ::snapshot::MIN_SUPPORTED_STATE_CHUNK_VERSION }
+	fn current_version(&self) -> u64 { ::snapshot::STATE_CHUNK_VERSION }
+}
+
+/// Used to build block chunks.
+struct PowWorker<'a> {
+	chain: &'a BlockChain,
+	// block, receipt rlp pairs.
+	rlps: VecDeque<Bytes>,
+	current_hash: H256,
+	writer: &'a mut ChunkSink<'a>,
+	preferred_size: usize,
+}
+
+impl<'a> PowWorker<'a> {
+	// Repeatedly fill the buffers and writes out chunks, moving backwards from starting block hash.
+	// Loops until we reach the first desired block, and writes out the remainder.
+	fn chunk_all(&mut self, snapshot_blocks: u64) -> Result<(), Error> {
+		let mut loaded_size = 0;
+		let mut last = self.current_hash;
+
+		let genesis_hash = self.chain.genesis_hash();
+
+		for _ in 0..snapshot_blocks {
+			if self.current_hash == genesis_hash { break }
+
+			let (block, receipts) = self.chain.block(&self.current_hash)
+				.and_then(|b| self.chain.block_receipts(&self.current_hash).map(|r| (b, r)))
+				.ok_or(Error::BlockNotFound(self.current_hash))?;
+
+			let abridged_rlp = AbridgedBlock::from_block_view(&block.view()).into_inner();
+
+			let pair = {
+				let mut pair_stream = RlpStream::new_list(2);
+				pair_stream.append_raw(&abridged_rlp, 1).append(&receipts);
+				pair_stream.out()
+			};
+
+			let new_loaded_size = loaded_size + pair.len();
+
+			// cut off the chunk if too large.
+
+			if new_loaded_size > self.preferred_size && !self.rlps.is_empty() {
+				self.write_chunk(last)?;
+				loaded_size = pair.len();
+			} else {
+				loaded_size = new_loaded_size;
+			}
+
+			self.rlps.push_front(pair);
+
+			last = self.current_hash;
+			self.current_hash = block.header_view().parent_hash();
+		}
+
+		if loaded_size != 0 {
+			self.write_chunk(last)?;
+		}
+
+		Ok(())
+	}
+
+	// write out the data in the buffers to a chunk on disk
+	//
+	// we preface each chunk with the parent of the first block's details,
+	// obtained from the details of the last block written.
+	fn write_chunk(&mut self, last: H256) -> Result<(), Error> {
+		trace!(target: "snapshot", "prepared block chunk with {} blocks", self.rlps.len());
+
+		let (last_header, last_details) = self.chain.block_header(&last)
+			.and_then(|n| self.chain.block_details(&last).map(|d| (n, d)))
+			.ok_or(Error::BlockNotFound(last))?;
+
+		let parent_number = last_header.number() - 1;
+		let parent_hash = last_header.parent_hash();
+		let parent_total_difficulty = last_details.total_difficulty - *last_header.difficulty();
+
+		trace!(target: "snapshot", "parent last written block: {}", parent_hash);
+
+		let num_entries = self.rlps.len();
+		let mut rlp_stream = RlpStream::new_list(3 + num_entries);
+		rlp_stream.append(&parent_number).append(parent_hash).append(&parent_total_difficulty);
+
+		for pair in self.rlps.drain(..) {
+			rlp_stream.append_raw(&pair, 1);
+		}
+
+		let raw_data = rlp_stream.out();
+
+		(self.writer)(&raw_data)?;
+
+		Ok(())
+	}
+}
+
+/// Rebuilder for proof-of-work chains.
+/// Does basic verification for all blocks, but `PoW` verification for some.
+/// Blocks must be fed in-order.
+///
+/// The first block in every chunk is disconnected from the last block in the
+/// chunk before it, as chunks may be submitted out-of-order.
+///
+/// After all chunks have been submitted, we "glue" the chunks together.
+pub struct PowRebuilder {
+	chain: BlockChain,
+	db: Arc<KeyValueDB>,
+	rng: OsRng,
+	disconnected: Vec<(u64, H256)>,
+	best_number: u64,
+	best_hash: H256,
+	best_root: H256,
+	fed_blocks: u64,
+	snapshot_blocks: u64,
+}
+
+impl PowRebuilder {
+	/// Create a new PowRebuilder.
+	fn new(chain: BlockChain, db: Arc<KeyValueDB>, manifest: &ManifestData, snapshot_blocks: u64) -> Result<Self, ::error::Error> {
+		Ok(PowRebuilder {
+			chain: chain,
+			db: db,
+			rng: OsRng::new()?,
+			disconnected: Vec::new(),
+			best_number: manifest.block_number,
+			best_hash: manifest.block_hash,
+			best_root: manifest.state_root,
+			fed_blocks: 0,
+			snapshot_blocks: snapshot_blocks,
+		})
+	}
+}
+
+impl Rebuilder for PowRebuilder {
+	/// Feed the rebuilder an uncompressed block chunk.
+	/// Returns the number of blocks fed or any errors.
+	fn feed(&mut self, chunk: &[u8], engine: &Engine, abort_flag: &AtomicBool) -> Result<(), ::error::Error> {
+		use basic_types::Seal::With;
+		use views::BlockView;
+		use snapshot::verify_old_block;
+		use util::U256;
+		use util::triehash::ordered_trie_root;
+
+		let rlp = UntrustedRlp::new(chunk);
+		let item_count = rlp.item_count()?;
+		let num_blocks = (item_count - 3) as u64;
+
+		trace!(target: "snapshot", "restoring block chunk with {} blocks.", item_count - 3);
+
+		if self.fed_blocks + num_blocks > self.snapshot_blocks {
+			return Err(Error::TooManyBlocks(self.snapshot_blocks, self.fed_blocks).into())
+		}
+
+		// todo: assert here that these values are consistent with chunks being in order.
+		let mut cur_number = rlp.val_at::<u64>(0)? + 1;
+		let mut parent_hash = rlp.val_at::<H256>(1)?;
+		let parent_total_difficulty = rlp.val_at::<U256>(2)?;
+
+		for idx in 3..item_count {
+			if !abort_flag.load(Ordering::SeqCst) { return Err(Error::RestorationAborted.into()) }
+
+			let pair = rlp.at(idx)?;
+			let abridged_rlp = pair.at(0)?.as_raw().to_owned();
+			let abridged_block = AbridgedBlock::from_raw(abridged_rlp);
+			let receipts: Vec<::receipt::Receipt> = pair.list_at(1)?;
+			let receipts_root = ordered_trie_root(
+				pair.at(1)?.iter().map(|r| r.as_raw().to_owned())
+			);
+
+			let block = abridged_block.to_block(parent_hash, cur_number, receipts_root)?;
+			let block_bytes = block.rlp_bytes(With);
+			let is_best = cur_number == self.best_number;
+
+			if is_best {
+				if block.header.hash() != self.best_hash {
+					return Err(Error::WrongBlockHash(cur_number, self.best_hash, block.header.hash()).into())
+				}
+
+				if block.header.state_root() != &self.best_root {
+					return Err(Error::WrongStateRoot(self.best_root, *block.header.state_root()).into())
+				}
+			}
+
+			verify_old_block(
+				&mut self.rng,
+				&block.header,
+				engine,
+				&self.chain,
+				Some(&block_bytes),
+				is_best
+			)?;
+
+			let mut batch = self.db.transaction();
+
+			// special-case the first block in each chunk.
+			if idx == 3 {
+				if self.chain.insert_unordered_block(&mut batch, &block_bytes, receipts, Some(parent_total_difficulty), is_best, false) {
+					self.disconnected.push((cur_number, block.header.hash()));
+				}
+			} else {
+				self.chain.insert_unordered_block(&mut batch, &block_bytes, receipts, None, is_best, false);
+			}
+			self.db.write_buffered(batch);
+			self.chain.commit();
+
+			parent_hash = BlockView::new(&block_bytes).hash();
+			cur_number += 1;
+		}
+
+		self.fed_blocks += num_blocks;
+
+		Ok(())
+	}
+
+	/// Glue together any disconnected chunks and check that the chain is complete.
+	fn finalize(&mut self, _: ::state_db::StateDB, _: &Engine) -> Result<(), ::error::Error> {
+		let mut batch = self.db.transaction();
+
+		for (first_num, first_hash) in self.disconnected.drain(..) {
+			let parent_num = first_num - 1;
+
+			// check if the parent is even in the chain.
+			// since we don't restore every single block in the chain,
+			// the first block of the first chunks has nothing to connect to.
+			if let Some(parent_hash) = self.chain.block_hash(parent_num) {
+				// if so, add the child to it.
+				self.chain.add_child(&mut batch, parent_hash, first_hash);
+			}
+		}
+
+		let genesis_hash = self.chain.genesis_hash();
+		self.chain.insert_epoch_transition(&mut batch, 0, ::blockchain::EpochTransition {
+			block_number: 0,
+			block_hash: genesis_hash,
+			proof: vec![],
+			state_proof: vec![],
+		});
+		self.db.write_buffered(batch);
+		Ok(())
+	}
+}

ethcore/src/snapshot/error.rs

@@ -59,6 +59,10 @@ pub enum Error {
 	ChunkTooSmall,
 	/// Snapshots not supported by the consensus engine.
 	SnapshotsUnsupported,
+	/// Bad epoch transition.
+	BadEpochProof(u64),
+	/// Wrong chunk format.
+	WrongChunkFormat(String),
 }
 
 impl fmt::Display for Error {
@@ -82,6 +86,8 @@ impl fmt::Display for Error {
 			Error::VersionNotSupported(ref ver) => write!(f, "Snapshot version {} is not supprted.", ver),
 			Error::ChunkTooSmall => write!(f, "Chunk size is too small."),
 			Error::SnapshotsUnsupported => write!(f, "Snapshots unsupported by consensus engine."),
+			Error::BadEpochProof(i) => write!(f, "Bad epoch proof for transition to epoch {}", i),
+			Error::WrongChunkFormat(ref msg) => write!(f, "Wrong chunk format: {}", msg),
 		}
 	}
 }

ethcore/src/snapshot/mod.rs

@@ -33,7 +33,7 @@ use util::{Bytes, Hashable, HashDB, DBValue, snappy, U256, Uint};
 use util::Mutex;
 use util::hash::{H256};
 use util::journaldb::{self, Algorithm, JournalDB};
-use util::kvdb::Database;
+use util::kvdb::KeyValueDB;
 use util::trie::{TrieDB, TrieDBMut, Trie, TrieMut};
 use util::sha3::SHA3_NULL_RLP;
 use rlp::{RlpStream, UntrustedRlp};
@@ -83,6 +83,11 @@ mod traits {
 // Try to have chunks be around 4MB (before compression)
 const PREFERRED_CHUNK_SIZE: usize = 4 * 1024 * 1024;
 
+// Minimum supported state chunk version.
+const MIN_SUPPORTED_STATE_CHUNK_VERSION: u64 = 1;
+// current state chunk version.
+const STATE_CHUNK_VERSION: u64 = 2;
+
 /// A progress indicator for snapshots.
 #[derive(Debug, Default)]
 pub struct Progress {
@@ -135,6 +140,7 @@ pub fn take_snapshot<W: SnapshotWriter + Send>(
 
 	let writer = Mutex::new(writer);
 	let chunker = engine.snapshot_components().ok_or(Error::SnapshotsUnsupported)?;
+	let snapshot_version = chunker.current_version();
 	let (state_hashes, block_hashes) = scope(|scope| {
 		let writer = &writer;
 		let block_guard = scope.spawn(move || chunk_secondary(chunker, chain, block_at, writer, p));
@@ -148,7 +154,7 @@ pub fn take_snapshot<W: SnapshotWriter + Send>(
 	info!("produced {} state chunks and {} block chunks.", state_hashes.len(), block_hashes.len());
 
 	let manifest_data = ManifestData {
-		version: 2,
+		version: snapshot_version,
 		state_hashes: state_hashes,
 		block_hashes: block_hashes,
 		state_root: *state_root,
@@ -309,7 +315,7 @@ pub struct StateRebuilder {
 
 impl StateRebuilder {
 	/// Create a new state rebuilder to write into the given backing DB.
-	pub fn new(db: Arc<Database>, pruning: Algorithm) -> Self {
+	pub fn new(db: Arc<KeyValueDB>, pruning: Algorithm) -> Self {
 		StateRebuilder {
 			db: journaldb::new(db.clone(), pruning, ::db::COL_STATE),
 			state_root: SHA3_NULL_RLP,
@@ -384,7 +390,7 @@ impl StateRebuilder {
 	/// Finalize the restoration. Check for accounts missing code and make a dummy
 	/// journal entry.
 	/// Once all chunks have been fed, there should be nothing missing.
-	pub fn finalize(mut self, era: u64, id: H256) -> Result<(), ::error::Error> {
+	pub fn finalize(mut self, era: u64, id: H256) -> Result<Box<JournalDB>, ::error::Error> {
 		let missing = self.missing_code.keys().cloned().collect::<Vec<_>>();
 		if !missing.is_empty() { return Err(Error::MissingCode(missing).into()) }
 
@@ -392,7 +398,7 @@ impl StateRebuilder {
 		self.db.journal_under(&mut batch, era, &id)?;
 		self.db.backing().write_buffered(batch);
 
-		Ok(())
+		Ok(self.db)
 	}
 
 	/// Get the state root of the rebuilder.

ethcore/src/snapshot/service.rs

@@ -106,6 +106,7 @@ impl Restoration {
 		let secondary = components.rebuilder(chain, raw_db.clone(), &manifest)?;
 
 		let root = manifest.state_root.clone();
+
 		Ok(Restoration {
 			manifest: manifest,
 			state_chunks_left: state_chunks,
@@ -150,7 +151,7 @@ impl Restoration {
 	}
 
 	// finish up restoration.
-	fn finalize(mut self) -> Result<(), Error> {
+	fn finalize(mut self, engine: &Engine) -> Result<(), Error> {
 		use util::trie::TrieError;
 
 		if !self.is_done() { return Ok(()) }
@@ -163,10 +164,11 @@ impl Restoration {
 		}
 
 		// check for missing code.
-		self.state.finalize(self.manifest.block_number, self.manifest.block_hash)?;
+		let db = self.state.finalize(self.manifest.block_number, self.manifest.block_hash)?;
+		let db = ::state_db::StateDB::new(db, 0);
 
 		// connect out-of-order chunks and verify chain integrity.
-		self.secondary.finalize()?;
+		self.secondary.finalize(db, engine)?;
 
 		if let Some(writer) = self.writer {
 			writer.finish(self.manifest)?;
@@ -450,7 +452,10 @@ impl Service {
 		let recover = rest.as_ref().map_or(false, |rest| rest.writer.is_some());
 
 		// destroy the restoration before replacing databases and snapshot.
-		rest.take().map(Restoration::finalize).unwrap_or(Ok(()))?;
+		rest.take()
+			.map(|r| r.finalize(&*self.engine))
+			.unwrap_or(Ok(()))?;
+
 		self.replace_client_db()?;
 
 		if recover {
@@ -554,6 +559,11 @@ impl SnapshotService for Service {
 		self.reader.read().as_ref().map(|r| r.manifest().clone())
 	}
 
+	fn min_supported_version(&self) -> Option<u64> {
+		self.engine.snapshot_components()
+			.map(|c| c.min_supported_version())
+	}
+
 	fn chunk(&self, hash: H256) -> Option<Bytes> {
 		self.reader.read().as_ref().and_then(|r| r.chunk(hash).ok())
 	}

ethcore/src/snapshot/snapshot_service_trait.rs

@@ -27,6 +27,10 @@ pub trait SnapshotService : Sync + Send {
 	/// Query the most recent manifest data.
 	fn manifest(&self) -> Option<ManifestData>;
 
+	/// Get the minimum supported snapshot version number.
+	/// `None` indicates warp sync isn't supported by the consensus engine.
+	fn min_supported_version(&self) -> Option<u64>;
+
 	/// Get raw chunk for a given hash.
 	fn chunk(&self, hash: H256) -> Option<Bytes>;
 

ethcore/src/snapshot/tests/helpers.rs

@@ -17,13 +17,24 @@
 //! Snapshot test helpers. These are used to build blockchains and state tries
 //! which can be queried before and after a full snapshot/restore cycle.
 
-use basic_account::BasicAccount;
+use std::sync::Arc;
+
 use account_db::AccountDBMut;
+use basic_account::BasicAccount;
+use blockchain::BlockChain;
+use client::{BlockChainClient, Client};
+use engines::Engine;
+use snapshot::{StateRebuilder};
+use snapshot::io::{SnapshotReader, PackedWriter, PackedReader};
+use state_db::StateDB;
+
+use devtools::{RandomTempPath, GuardedTempResult};
 use rand::Rng;
 
-use util::DBValue;
+use util::{DBValue, KeyValueDB};
 use util::hash::H256;
 use util::hashdb::HashDB;
+use util::journaldb;
 use util::trie::{Alphabet, StandardMap, SecTrieDBMut, TrieMut, ValueMode};
 use util::trie::{TrieDB, TrieDBMut, Trie};
 use util::sha3::SHA3_NULL_RLP;
@@ -125,3 +136,67 @@ pub fn compare_dbs(one: &HashDB, two: &HashDB) {
 		assert_eq!(one.get(&key).unwrap(), two.get(&key).unwrap());
 	}
 }
+
+/// Take a snapshot from the given client into a temporary file.
+/// Return a snapshot reader for it.
+pub fn snap(client: &Client) -> GuardedTempResult<Box<SnapshotReader>> {
+	use ids::BlockId;
+
+	let dir = RandomTempPath::new();
+	let writer = PackedWriter::new(dir.as_path()).unwrap();
+	let progress = Default::default();
+
+	let hash = client.chain_info().best_block_hash;
+	client.take_snapshot(writer, BlockId::Hash(hash), &progress).unwrap();
+
+	let reader = PackedReader::new(dir.as_path()).unwrap().unwrap();
+
+	GuardedTempResult {
+		result: Some(Box::new(reader)),
+		_temp: dir,
+	}
+}
+
+/// Restore a snapshot into a given database. This will read chunks from the given reader
+/// write into the given database.
+pub fn restore(
+	db: Arc<KeyValueDB>,
+	engine: &Engine,
+	reader: &SnapshotReader,
+	genesis: &[u8],
+) -> Result<(), ::error::Error> {
+	use std::sync::atomic::AtomicBool;
+	use util::snappy;
+
+	let flag = AtomicBool::new(true);
+	let components = engine.snapshot_components().unwrap();
+	let manifest = reader.manifest();
+
+	let mut state = StateRebuilder::new(db.clone(), journaldb::Algorithm::Archive);
+	let mut secondary = {
+		let chain = BlockChain::new(Default::default(), genesis, db.clone());
+		components.rebuilder(chain, db, manifest).unwrap()
+	};
+
+	let mut snappy_buffer = Vec::new();
+
+	trace!(target: "snapshot", "restoring state");
+	for state_chunk_hash in manifest.state_hashes.iter() {
+		trace!(target: "snapshot", "state chunk hash: {}", state_chunk_hash);
+		let chunk = reader.chunk(*state_chunk_hash).unwrap();
+		let len = snappy::decompress_into(&chunk, &mut snappy_buffer).unwrap();
+		state.feed(&snappy_buffer[..len], &flag)?;
+	}
+
+	trace!(target: "snapshot", "restoring secondary");
+	for chunk_hash in manifest.block_hashes.iter() {
+		let chunk = reader.chunk(*chunk_hash).unwrap();
+		let len = snappy::decompress_into(&chunk, &mut snappy_buffer).unwrap();
+		secondary.feed(&snappy_buffer[..len], engine, &flag)?;
+	}
+
+	let jdb = state.finalize(manifest.block_number, manifest.block_hash)?;
+	let state_db = StateDB::new(jdb, 0);
+
+	secondary.finalize(state_db, engine)
+}

ethcore/src/snapshot/tests/mod.rs

@@ -16,7 +16,8 @@
 
 //! Snapshot tests.
 
-mod blocks;
+mod proof_of_work;
+mod proof_of_authority;
 mod state;
 mod service;
 

ethcore/src/snapshot/tests/proof_of_authority.rs

@@ -0,0 +1,249 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! PoA block chunker and rebuilder tests.
+
+use std::cell::RefCell;
+use std::sync::Arc;
+use std::str::FromStr;
+
+use account_provider::AccountProvider;
+use client::{Client, BlockChainClient, MiningBlockChainClient};
+use ethkey::Secret;
+use engines::Seal;
+use futures::Future;
+use miner::MinerService;
+use native_contracts::test_contracts::ValidatorSet;
+use snapshot::tests::helpers as snapshot_helpers;
+use spec::Spec;
+use tests::helpers;
+use transaction::{Transaction, Action, SignedTransaction};
+
+use util::{Address, Hashable};
+use util::kvdb;
+
+const PASS: &'static str = "";
+const TRANSITION_BLOCK_1: usize = 2; // block at which the contract becomes activated.
+const TRANSITION_BLOCK_2: usize = 6; // block at which the second contract activates.
+
+macro_rules! secret {
+	($e: expr) => { Secret::from_slice(&$e.sha3()).expect(format!("sha3({}) not valid secret.", $e).as_str()) }
+}
+
+lazy_static! {
+	// contract addresses.
+	static ref CONTRACT_ADDR_1: Address = Address::from_str("0000000000000000000000000000000000000005").unwrap();
+	static ref CONTRACT_ADDR_2: Address = Address::from_str("0000000000000000000000000000000000000006").unwrap();
+	// secret: `sha3(1)`, and initial validator.
+	static ref RICH_ADDR: Address = Address::from_str("7d577a597b2742b498cb5cf0c26cdcd726d39e6e").unwrap();
+	// rich address' secret.
+	static ref RICH_SECRET: Secret = secret!("1");
+}
+
+
+/// Contract code used here: https://gist.github.com/rphmeier/2de14fd365a969e3a9e10d77eb9a1e37
+/// Account with secrets "1".sha3() is initially the validator.
+/// Transitions to the contract at block 2, initially same validator set.
+/// Create a new Spec with BasicAuthority which uses a contract at address 5 to determine the current validators using `getValidators`.
+/// `native_contracts::test_contracts::ValidatorSet` provides a native wrapper for the ABi.
+fn spec_fixed_to_contract() -> Spec {
+	let data = include_bytes!("test_validator_contract.json");
+	Spec::load(&data[..]).unwrap()
+}
+
+// creates an account provider, filling it with accounts from all the given
+// secrets and password `PASS`.
+// returns addresses corresponding to secrets.
+fn make_accounts(secrets: &[Secret]) -> (Arc<AccountProvider>, Vec<Address>) {
+	let provider = AccountProvider::transient_provider();
+
+	let addrs = secrets.iter()
+		.cloned()
+		.map(|s| provider.insert_account(s, PASS).unwrap())
+		.collect();
+
+	(Arc::new(provider), addrs)
+}
+
+// validator transition. block number and new validators. must be after `TRANSITION_BLOCK`.
+// all addresses in the set must be in the account provider.
+enum Transition {
+	// manual transition via transaction
+	Manual(usize, Vec<Address>),
+	// implicit transition via multi-set
+	Implicit(usize, Vec<Address>),
+}
+
+// create a chain with the given transitions and some blocks beyond that transition.
+fn make_chain(accounts: Arc<AccountProvider>, blocks_beyond: usize, transitions: Vec<Transition>) -> Arc<Client> {
+	let client = helpers::generate_dummy_client_with_spec_and_accounts(
+		spec_fixed_to_contract, Some(accounts.clone()));
+
+	let mut cur_signers = vec![*RICH_ADDR];
+	{
+		let engine = client.engine();
+		engine.register_client(Arc::downgrade(&client));
+	}
+
+	{
+		// push a block with given number, signed by one of the signers, with given transactions.
+		let push_block = |signers: &[Address], n, txs: Vec<SignedTransaction>| {
+			use block::IsBlock;
+
+			let engine = client.engine();
+			let idx = n as usize % signers.len();
+			engine.set_signer(accounts.clone(), signers[idx], PASS.to_owned());
+
+			trace!(target: "snapshot", "Pushing block #{}, {} txs, author={}", n, txs.len(), signers[idx]);
+
+			let mut open_block = client.prepare_open_block(signers[idx], (5_000_000.into(), 5_000_000.into()), Vec::new());
+			for tx in txs {
+				open_block.push_transaction(tx, None).unwrap();
+			}
+			let block = open_block.close_and_lock();
+			let seal = match engine.generate_seal(block.block()) {
+				Seal::Regular(seal) => seal,
+				_ => panic!("Unable to generate seal for dummy chain block #{}", n),
+			};
+			let block = block.seal(&*engine, seal).unwrap();
+
+			client.import_sealed_block(block).unwrap();
+		};
+
+		// execution callback for native contract: push transaction to be sealed.
+		let nonce = RefCell::new(client.engine().account_start_nonce());
+		let exec = |addr, data| {
+			let mut nonce = nonce.borrow_mut();
+			let transaction = Transaction {
+				nonce: *nonce,
+				gas_price: 0.into(),
+				gas: 1_000_000.into(),
+				action: Action::Call(addr),
+				value: 0.into(),
+				data: data,
+			}.sign(&*RICH_SECRET, client.signing_network_id());
+
+			client.miner().import_own_transaction(&*client, transaction.into()).unwrap();
+
+			*nonce = *nonce + 1.into();
+			Ok(Vec::new())
+		};
+
+		let contract_1 = ValidatorSet::new(*CONTRACT_ADDR_1);
+		let contract_2 = ValidatorSet::new(*CONTRACT_ADDR_2);
+
+		// apply all transitions.
+		for transition in transitions {
+			let (num, manual, new_set) = match transition {
+				Transition::Manual(num, new_set) => (num, true, new_set),
+				Transition::Implicit(num, new_set) => (num, false, new_set),
+			};
+
+			if num < TRANSITION_BLOCK_1 {
+				panic!("Bad test: issued epoch change before transition to contract.");
+			}
+
+			for number in client.chain_info().best_block_number + 1 .. num as u64 {
+				push_block(&cur_signers, number, vec![]);
+			}
+
+			let pending = if manual {
+				trace!(target: "snapshot", "applying set transition at block #{}", num);
+				let contract = match num >= TRANSITION_BLOCK_2 {
+					true => &contract_2,
+					false => &contract_1,
+				};
+
+				contract.set_validators(&exec, new_set.clone()).wait().unwrap();
+				client.ready_transactions()
+					.into_iter()
+					.map(|x| x.transaction)
+					.collect()
+			} else {
+				Vec::new()
+			};
+
+			push_block(&cur_signers, num as u64, pending);
+			cur_signers = new_set;
+		}
+
+		// make blocks beyond.
+		for number in (client.chain_info().best_block_number..).take(blocks_beyond) {
+			push_block(&cur_signers, number + 1, vec![]);
+		}
+	}
+
+	client
+}
+
+#[test]
+fn fixed_to_contract() {
+	let (provider, addrs) = make_accounts(&[
+		RICH_SECRET.clone(),
+		secret!("foo"),
+		secret!("bar"),
+		secret!("test"),
+		secret!("signer"),
+		secret!("crypto"),
+		secret!("wizard"),
+		secret!("dog42"),
+	]);
+
+	assert!(provider.has_account(*RICH_ADDR).unwrap());
+
+	let client = make_chain(provider, 1, vec![
+		Transition::Manual(3, vec![addrs[2], addrs[3], addrs[5], addrs[7]]),
+		Transition::Manual(4, vec![addrs[0], addrs[1], addrs[4], addrs[6]]),
+	]);
+
+	assert_eq!(client.chain_info().best_block_number, 5);
+	let reader = snapshot_helpers::snap(&*client);
+
+	let new_db = kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0));
+	let spec = spec_fixed_to_contract();
+
+	snapshot_helpers::restore(Arc::new(new_db), &*spec.engine, &**reader, &spec.genesis_block()).unwrap();
+}
+
+#[test]
+fn fixed_to_contract_to_contract() {
+	let (provider, addrs) = make_accounts(&[
+		RICH_SECRET.clone(),
+		secret!("foo"),
+		secret!("bar"),
+		secret!("test"),
+		secret!("signer"),
+		secret!("crypto"),
+		secret!("wizard"),
+		secret!("dog42"),
+	]);
+
+	assert!(provider.has_account(*RICH_ADDR).unwrap());
+
+	let client = make_chain(provider, 2, vec![
+		Transition::Manual(3, vec![addrs[2], addrs[3], addrs[5], addrs[7]]),
+		Transition::Manual(4, vec![addrs[0], addrs[1], addrs[4], addrs[6]]),
+		Transition::Implicit(5, vec![addrs[0]]),
+		Transition::Manual(8, vec![addrs[2], addrs[4], addrs[6], addrs[7]]),
+	]);
+
+	assert_eq!(client.chain_info().best_block_number, 10);
+	let reader = snapshot_helpers::snap(&*client);
+	let new_db = kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0));
+	let spec = spec_fixed_to_contract();
+
+	snapshot_helpers::restore(Arc::new(new_db), &*spec.engine, &**reader, &spec.genesis_block()).unwrap();
+}

ethcore/src/snapshot/tests/proof_of_work.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Block chunker and rebuilder tests.
+//! PoW block chunker and rebuilder tests.
 
 use devtools::RandomTempPath;
 use error::Error;
@@ -23,8 +23,10 @@ use blockchain::generator::{ChainGenerator, ChainIterator, BlockFinalizer};
 use blockchain::BlockChain;
 use snapshot::{chunk_secondary, Error as SnapshotError, Progress, SnapshotComponents};
 use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter};
+use state_db::StateDB;
 
 use util::{Mutex, snappy};
+use util::journaldb::{self, Algorithm};
 use util::kvdb::{self, KeyValueDB, DBTransaction};
 
 use std::sync::Arc;
@@ -81,6 +83,7 @@ fn chunk_and_restore(amount: u64) {
 	// restore it.
 	let new_db = Arc::new(kvdb::in_memory(::db::NUM_COLUMNS.unwrap_or(0)));
 	let new_chain = BlockChain::new(Default::default(), &genesis, new_db.clone());
+	let new_state = StateDB::new(journaldb::new(new_db.clone(), Algorithm::Archive, None), 0);
 	let mut rebuilder = SNAPSHOT_MODE.rebuilder(new_chain, new_db.clone(), &manifest).unwrap();
 
 	let reader = PackedReader::new(&snapshot_path).unwrap().unwrap();
@@ -91,7 +94,7 @@ fn chunk_and_restore(amount: u64) {
 		rebuilder.feed(&chunk, engine.as_ref(), &flag).unwrap();
 	}
 
-	rebuilder.finalize().unwrap();
+	rebuilder.finalize(new_state, engine.as_ref()).unwrap();
 	drop(rebuilder);
 
 	// and test it.

ethcore/src/snapshot/tests/test_validator_contract.json

@@ -0,0 +1,51 @@
+{
+	"name": "TestValidatorContract",
+	"engine": {
+		"basicAuthority": {
+			"params": {
+				"gasLimitBoundDivisor": "0x0400",
+				"durationLimit": "0x0d",
+				"validators": {
+					"multi": {
+						"0": { "list": ["0x7d577a597b2742b498cb5cf0c26cdcd726d39e6e"] },
+						"2": { "contract": "0x0000000000000000000000000000000000000005" },
+						"6": { "contract": "0x0000000000000000000000000000000000000006" }
+					}
+				}
+			}
+		}
+	},
+	"params": {
+		"accountStartNonce": "0x0",
+		"maximumExtraDataSize": "0x20",
+		"minGasLimit": "0x1388",
+		"networkID" : "0x69"
+	},
+	"genesis": {
+		"seal": {
+			"generic": "0xc180"
+		},
+		"difficulty": "0x20000",
+		"author": "0x0000000000000000000000000000000000000000",
+		"timestamp": "0x00",
+		"parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+		"extraData": "0x",
+		"gasLimit": "0x2fefd8"
+	},
+	"accounts": {
+		"0000000000000000000000000000000000000001": { "balance": "1", "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } },
+		"0000000000000000000000000000000000000002": { "balance": "1", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } },
+		"0000000000000000000000000000000000000003": { "balance": "1", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } },
+		"0000000000000000000000000000000000000004": { "balance": "1", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } },
+		"0000000000000000000000000000000000000005": {
+			"balance": "1",
+			"constructor": "6060604052604060405190810160405280737d577a597b2742b498cb5cf0c26cdcd726d39e6e73ffffffffffffffffffffffffffffffffffffffff1681526020017382a978b3f5962a5b0957d9ee9eef472ee55b42f173ffffffffffffffffffffffffffffffffffffffff16815250600290600261007e929190610096565b50341561008757fe5b5b60006001819055505b610163565b82805482825590600052602060002090810192821561010f579160200282015b8281111561010e5782518260006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550916020019190600101906100b6565b5b50905061011c9190610120565b5090565b61016091905b8082111561015c57600081816101000a81549073ffffffffffffffffffffffffffffffffffffffff021916905550600101610126565b5090565b90565b61045d806101726000396000f30060606040526000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063303e98e5146100675780639300c9261461008d578063b7ab4db5146100e4578063bfc708a014610159578063fd6e1b501461018f575bfe5b341561006f57fe5b6100776101c5565b6040518082815260200191505060405180910390f35b341561009557fe5b6100e26004808035906020019082018035906020019080806020026020016040519081016040528093929190818152602001838360200280828437820191505050505050919050506101d0565b005b34156100ec57fe5b6100f46102b3565b6040518080602001828103825283818151815260200191508051906020019060200280838360008314610146575b80518252602083111561014657602082019150602081019050602083039250610122565b5050509050019250505060405180910390f35b341561016157fe5b61018d600480803573ffffffffffffffffffffffffffffffffffffffff16906020019091905050610348565b005b341561019757fe5b6101c3600480803573ffffffffffffffffffffffffffffffffffffffff1690602001909190505061034c565b005b600060015490505b90565b600081600290805190602001906101e8929190610350565b50600143034090506000546000191681600019161415156102ae578060008160001916905550600160016000828254019250508190555060015481600019167f47e91f47ccfdcb578564e1af55da55c5e5d33403372fe68e4fed3dfd385764a184604051808060200182810382528381815181526020019150805190602001906020028083836000831461029b575b80518252602083111561029b57602082019150602081019050602083039250610277565b5050509050019250505060405180910390a35b5b5050565b6102bb6103da565b600280548060200260200160405190810160405280929190818152602001828054801561033d57602002820191906000526020600020905b8160009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190600101908083116102f3575b505050505090505b90565b5b50565b5b50565b8280548282559060005260206000209081019282156103c9579160200282015b828111156103c85782518260006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555091602001919060010190610370565b5b5090506103d691906103ee565b5090565b602060405190810160405280600081525090565b61042e91905b8082111561042a57600081816101000a81549073ffffffffffffffffffffffffffffffffffffffff0219169055506001016103f4565b5090565b905600a165627a7a723058205c9ed1e1da2b93682907ac47377a662b21a5f9d89c4b21be40b098bdb00254360029"
+		},
+		"0000000000000000000000000000000000000006": {
+			"balance": "1",
+			"constructor": "6060604052602060405190810160405280737d577a597b2742b498cb5cf0c26cdcd726d39e6e73ffffffffffffffffffffffffffffffffffffffff16815250600290600161004e929190610066565b50341561005757fe5b5b60006001819055505b610133565b8280548282559060005260206000209081019282156100df579160200282015b828111156100de5782518260006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555091602001919060010190610086565b5b5090506100ec91906100f0565b5090565b61013091905b8082111561012c57600081816101000a81549073ffffffffffffffffffffffffffffffffffffffff0219169055506001016100f6565b5090565b90565b61045d806101426000396000f30060606040526000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063303e98e5146100675780639300c9261461008d578063b7ab4db5146100e4578063bfc708a014610159578063fd6e1b501461018f575bfe5b341561006f57fe5b6100776101c5565b6040518082815260200191505060405180910390f35b341561009557fe5b6100e26004808035906020019082018035906020019080806020026020016040519081016040528093929190818152602001838360200280828437820191505050505050919050506101d0565b005b34156100ec57fe5b6100f46102b3565b6040518080602001828103825283818151815260200191508051906020019060200280838360008314610146575b80518252602083111561014657602082019150602081019050602083039250610122565b5050509050019250505060405180910390f35b341561016157fe5b61018d600480803573ffffffffffffffffffffffffffffffffffffffff16906020019091905050610348565b005b341561019757fe5b6101c3600480803573ffffffffffffffffffffffffffffffffffffffff1690602001909190505061034c565b005b600060015490505b90565b600081600290805190602001906101e8929190610350565b50600143034090506000546000191681600019161415156102ae578060008160001916905550600160016000828254019250508190555060015481600019167f47e91f47ccfdcb578564e1af55da55c5e5d33403372fe68e4fed3dfd385764a184604051808060200182810382528381815181526020019150805190602001906020028083836000831461029b575b80518252602083111561029b57602082019150602081019050602083039250610277565b5050509050019250505060405180910390a35b5b5050565b6102bb6103da565b600280548060200260200160405190810160405280929190818152602001828054801561033d57602002820191906000526020600020905b8160009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190600101908083116102f3575b505050505090505b90565b5b50565b5b50565b8280548282559060005260206000209081019282156103c9579160200282015b828111156103c85782518260006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555091602001919060010190610370565b5b5090506103d691906103ee565b5090565b602060405190810160405280600081525090565b61042e91905b8082111561042a57600081816101000a81549073ffffffffffffffffffffffffffffffffffffffff0219169055506001016103f4565b5090565b905600a165627a7a723058203070810251dcb89c9838d957eb3dbeef357bef0902e0245e3dc3849b6143c3960029"
+		},
+		"0x7d577a597b2742b498cb5cf0c26cdcd726d39e6e": { "balance": "1606938044258990275541962092341162602522202993782792835301376" },
+		"0x82a978b3f5962a5b0957d9ee9eef472ee55b42f1": { "balance": "1606938044258990275541962092341162602522202993782792835301376" }
+	}
+}

ethcore/src/state/mod.rs

@@ -189,7 +189,7 @@ pub fn check_proof(
 		Err(_) => return ProvedExecution::BadProof,
 	};
 
-	match state.execute(env_info, engine, transaction, false) {
+	match state.execute(env_info, engine, transaction, false, true) {
 		Ok(executed) => ProvedExecution::Complete(executed),
 		Err(ExecutionError::Internal(_)) => ProvedExecution::BadProof,
 		Err(e) => ProvedExecution::Failed(e),
@@ -604,7 +604,7 @@ impl<B: Backend> State<B> {
 	pub fn apply(&mut self, env_info: &EnvInfo, engine: &Engine, t: &SignedTransaction, tracing: bool) -> ApplyResult {
 //		let old = self.to_pod();
 
-		let e = self.execute(env_info, engine, t, tracing)?;
+		let e = self.execute(env_info, engine, t, tracing, false)?;
 //		trace!("Applied transaction. Diff:\n{}\n", state_diff::diff_pod(&old, &self.to_pod()));
 		let state_root = if env_info.number < engine.params().eip98_transition || env_info.number < engine.params().validate_receipts_transition {
 			self.commit()?;
@@ -617,12 +617,22 @@ impl<B: Backend> State<B> {
 		Ok(ApplyOutcome{receipt: receipt, trace: e.trace})
 	}
 
-	// Execute a given transaction.
-	fn execute(&mut self, env_info: &EnvInfo, engine: &Engine, t: &SignedTransaction, tracing: bool) -> Result<Executed, ExecutionError> {
+	// Execute a given transaction without committing changes.
+	//
+	// `virt` signals that we are executing outside of a block set and restrictions like
+	// gas limits and gas costs should be lifted.
+	fn execute(&mut self, env_info: &EnvInfo, engine: &Engine, t: &SignedTransaction, tracing: bool, virt: bool)
+		-> Result<Executed, ExecutionError>
+	{
 		let options = TransactOptions { tracing: tracing, vm_tracing: false, check_nonce: true };
 		let vm_factory = self.factories.vm.clone();
 
-		Executive::new(self, env_info, engine, &vm_factory).transact(t, options)
+		let mut e = Executive::new(self, env_info, engine, &vm_factory);
+
+		match virt {
+			true => e.transact_virtual(t, options),
+			false => e.transact(t, options),
+		}
 	}
 
 

rpc/src/v1/tests/helpers/snapshot_service.rs

@@ -41,6 +41,7 @@ impl TestSnapshotService {
 
 impl SnapshotService for TestSnapshotService {
 	fn manifest(&self) -> Option<ManifestData> { None }
+	fn min_supported_version(&self) -> Option<u64> { None }
 	fn chunk(&self, _hash: H256) -> Option<Bytes> { None }
 	fn status(&self) -> RestorationStatus { self.status.lock().clone() }
 	fn begin_restore(&self, _manifest: ManifestData) { }

sync/src/chain.rs

@@ -158,8 +158,6 @@ pub const SNAPSHOT_SYNC_PACKET_COUNT: u8 = 0x16;
 
 const MAX_SNAPSHOT_CHUNKS_DOWNLOAD_AHEAD: usize = 3;
 
-const MIN_SUPPORTED_SNAPSHOT_MANIFEST_VERSION: u64 = 1;
-
 const WAIT_PEERS_TIMEOUT_SEC: u64 = 5;
 const STATUS_TIMEOUT_SEC: u64 = 5;
 const HEADERS_TIMEOUT_SEC: u64 = 15;
@@ -504,7 +502,7 @@ impl ChainSync {
 	}
 
 	fn maybe_start_snapshot_sync(&mut self, io: &mut SyncIo) {
-		if !self.enable_warp_sync {
+		if !self.enable_warp_sync || io.snapshot_service().min_supported_version().is_none() {
 			return;
 		}
 		if self.state != SyncState::WaitingPeers && self.state != SyncState::Blocks && self.state != SyncState::Waiting {
@@ -1042,7 +1040,11 @@ impl ChainSync {
 			}
 			Ok(manifest) => manifest,
 		};
-		if manifest.version < MIN_SUPPORTED_SNAPSHOT_MANIFEST_VERSION {
+
+		let is_supported_version = io.snapshot_service().min_supported_version()
+			.map_or(false, |v| manifest.version >= v);
+
+		if !is_supported_version {
 			trace!(target: "sync", "{}: Snapshot manifest version too low: {}", peer_id, manifest.version);
 			io.disable_peer(peer_id);
 			self.continue_sync(io);

sync/src/tests/snapshot.rs

@@ -71,6 +71,10 @@ impl SnapshotService for TestSnapshotService {
 		self.manifest.as_ref().cloned()
 	}
 
+	fn min_supported_version(&self) -> Option<u64> {
+		Some(1)
+	}
+
 	fn chunk(&self, hash: H256) -> Option<Bytes> {
 		self.chunks.get(&hash).cloned()
 	}