PoA warp implementation (#5488)

* separate modules for consensus snapshot chunks

* bulk of authority warp logic

* finish authority warp implementation

* squash warnings and enable authority snapshot mode

* test harness for PoA

* fiddle with harness

* epoch generation proof fixes

* update constructor code

* store epoch transition proof after block commit

* basic snap and restore test

* use keyvaluedb in state restoration

* decompress chunks

* fix encoding issues

* fixed-to-contract-to-contract test

* implement ancient block import

* restore genesis transition in PoW snapshot

* add format version method to snapshot components

* supported version numbers in snapshot_components

* allow returning of ancient epoch transitions

* genesis hash mismatch check

* remove commented code

ethcore/src/client/ancient_import.rs

@@ -0,0 +1,68 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Helper for ancient block import.
+
+use std::sync::Arc;
+
+use engines::{Engine, EpochVerifier, EpochChange};
+use error::Error;
+use header::Header;
+
+use rand::Rng;
+use util::RwLock;
+
+// do "heavy" verification on ~1/50 blocks, randomly sampled.
+const HEAVY_VERIFY_RATE: f32 = 0.02;
+
+/// Ancient block verifier: import an ancient sequence of blocks in order from a starting
+/// epoch.
+pub struct AncientVerifier {
+	cur_verifier: RwLock<Box<EpochVerifier>>,
+	engine: Arc<Engine>,
+}
+
+impl AncientVerifier {
+	/// Create a new ancient block verifier with the given engine and initial verifier.
+	pub fn new(engine: Arc<Engine>, start_verifier: Box<EpochVerifier>) -> Self {
+		AncientVerifier {
+			cur_verifier: RwLock::new(start_verifier),
+			engine: engine,
+		}
+	}
+
+	/// Verify the next block header, randomly choosing whether to do heavy or light
+	/// verification. If the block is the end of an epoch, updates the epoch verifier.
+	pub fn verify<R: Rng, F: Fn(u64) -> Result<Box<EpochVerifier>, Error>>(
+		&self,
+		rng: &mut R,
+		header: &Header,
+		block: &[u8],
+		receipts: &[::receipt::Receipt],
+		load_verifier: F,
+	) -> Result<(), ::error::Error> {
+		match rng.gen::<f32>() <= HEAVY_VERIFY_RATE {
+			true => self.cur_verifier.read().verify_heavy(header)?,
+			false => self.cur_verifier.read().verify_light(header)?,
+		}
+
+		if let EpochChange::Yes(num) = self.engine.is_epoch_end(header, Some(block), Some(receipts)) {
+			*self.cur_verifier.write() = load_verifier(num)?;
+		}
+
+		Ok(())
+	}
+}

ethcore/src/client/client.rs

@@ -34,6 +34,7 @@ use basic_types::Seal;
 use block::*;
 use blockchain::{BlockChain, BlockProvider, EpochTransition, TreeRoute, ImportRoute};
 use blockchain::extras::TransactionAddress;
+use client::ancient_import::AncientVerifier;
 use client::Error as ClientError;
 use client::{
 	BlockId, TransactionId, UncleId, TraceId, ClientConfig, BlockChainClient,
@@ -61,7 +62,7 @@ use service::ClientIoMessage;
 use snapshot::{self, io as snapshot_io};
 use spec::Spec;
 use state_db::StateDB;
-use state::{self, State, CleanupMode};
+use state::{self, State};
 use trace;
 use trace::{TraceDB, ImportRequest as TraceImportRequest, LocalizedTrace, Database as TraceDatabase};
 use trace::FlatTransactionTraces;
@@ -152,6 +153,7 @@ pub struct Client {
 	factories: Factories,
 	history: u64,
 	rng: Mutex<OsRng>,
+	ancient_verifier: Mutex<Option<AncientVerifier>>,
 	on_user_defaults_change: Mutex<Option<Box<FnMut(Option<Mode>) + 'static + Send>>>,
 	registrar: Mutex<Option<Registry>>,
 	exit_handler: Mutex<Option<Box<Fn(bool, Option<String>) + 'static + Send>>>,
@@ -241,6 +243,7 @@ impl Client {
 			factories: factories,
 			history: history,
 			rng: Mutex::new(OsRng::new().map_err(::util::UtilError::StdIo)?),
+			ancient_verifier: Mutex::new(None),
 			on_user_defaults_change: Mutex::new(None),
 			registrar: Mutex::new(None),
 			exit_handler: Mutex::new(None),
@@ -256,7 +259,11 @@ impl Client {
 		// ensure genesis epoch proof in the DB.
 		{
 			let chain = client.chain.read();
-			client.generate_epoch_proof(&spec.genesis_header(), 0, &*chain);
+			let gh = spec.genesis_header();
+			if chain.epoch_transition(0, spec.genesis_header().hash()).is_none() {
+				trace!(target: "client", "No genesis transition found.");
+				client.generate_epoch_proof(&gh, 0, &*chain);
+			}
 		}
 
 		if let Some(reg_addr) = client.additional_params().get("registrar").and_then(|s| Address::from_str(s).ok()) {
@@ -540,25 +547,56 @@ impl Client {
 	fn import_old_block(&self, block_bytes: Bytes, receipts_bytes: Bytes) -> Result<H256, ::error::Error> {
 		let block = BlockView::new(&block_bytes);
 		let header = block.header();
+		let receipts = ::rlp::decode_list(&receipts_bytes);
 		let hash = header.hash();
 		let _import_lock = self.import_lock.lock();
+
 		{
 			let _timer = PerfTimer::new("import_old_block");
-			let mut rng = self.rng.lock();
 			let chain = self.chain.read();
+			let mut ancient_verifier = self.ancient_verifier.lock();
 
-			// verify block.
-			::snapshot::verify_old_block(
-				&mut *rng,
-				&header,
-				&*self.engine,
-				&*chain,
-				Some(&block_bytes),
-				false,
-			)?;
+			{
+				// closure for verifying a block.
+				let verify_with = |verifier: &AncientVerifier| -> Result<(), ::error::Error> {
+					// verify the block, passing a closure used to load an epoch verifier
+					// by number.
+					verifier.verify(
+						&mut *self.rng.lock(),
+						&header,
+						&block_bytes,
+						&receipts,
+						|epoch_num| chain.epoch_transition(epoch_num, hash)
+							.ok_or(BlockError::UnknownEpochTransition(epoch_num))
+							.map_err(Into::into)
+							.and_then(|t| self.engine.epoch_verifier(&header, &t.proof))
+					)
+				};
+
+				// initialize the ancient block verifier if we don't have one already.
+				match &mut *ancient_verifier {
+					&mut Some(ref verifier) => {
+						verify_with(verifier)?
+					}
+					x @ &mut None => {
+						// load most recent epoch.
+						trace!(target: "client", "Initializing ancient block restoration.");
+						let current_epoch_data = chain.epoch_transitions()
+							.take_while(|&(_, ref t)| t.block_number < header.number())
+							.last()
+							.map(|(_, t)| t.proof)
+							.expect("At least one epoch entry (genesis) always stored; qed");
+
+						let current_verifier = self.engine.epoch_verifier(&header, &current_epoch_data)?;
+						let current_verifier = AncientVerifier::new(self.engine.clone(), current_verifier);
+
+						verify_with(&current_verifier)?;
+						*x = Some(current_verifier);
+					}
+				}
+			}
 
 			// Commit results
-			let receipts = ::rlp::decode_list(&receipts_bytes);
 			let mut batch = DBTransaction::new();
 			chain.insert_unordered_block(&mut batch, &block_bytes, receipts, None, false, true);
 			// Final commit to the DB
@@ -590,7 +628,7 @@ impl Client {
 		let entering_new_epoch = {
 			use engines::EpochChange;
 			match self.engine.is_epoch_end(block.header(), Some(block_data), Some(&receipts)) {
-				EpochChange::Yes(e, _) => Some((block.header().clone(), e)),
+				EpochChange::Yes(e) => Some((block.header().clone(), e)),
 				EpochChange::No => None,
 				EpochChange::Unsure(_) => {
 					warn!(target: "client", "Detected invalid engine implementation.");
@@ -641,7 +679,8 @@ impl Client {
 
 		let mut batch = DBTransaction::new();
 		let hash = header.hash();
-		debug!(target: "client", "Generating validation proof for block {}", hash);
+		debug!(target: "client", "Generating validation proof for epoch {} at block {}",
+			epoch_number, hash);
 
 		// proof is two-part. state items read in lexicographical order,
 		// and the secondary "proof" part.
@@ -880,8 +919,8 @@ impl Client {
 		let start_hash = match at {
 			BlockId::Latest => {
 				let start_num = match db.earliest_era() {
-					Some(era) => ::std::cmp::max(era, best_block_number - history),
-					None => best_block_number - history,
+					Some(era) => ::std::cmp::max(era, best_block_number.saturating_sub(history)),
+					None => best_block_number.saturating_sub(history),
 				};
 
 				match self.block_hash(BlockId::Number(start_num)) {
@@ -992,16 +1031,9 @@ impl BlockChainClient for Client {
 		let mut state = self.state_at(block).ok_or(CallError::StatePruned)?;
 		let original_state = if analytics.state_diffing { Some(state.clone()) } else { None };
 
-		let sender = t.sender();
-		let balance = state.balance(&sender).map_err(|_| CallError::StateCorrupt)?;
-		let needed_balance = t.value + t.gas * t.gas_price;
-		if balance < needed_balance {
-			// give the sender a sufficient balance
-			state.add_balance(&sender, &(needed_balance - balance), CleanupMode::NoEmpty)
-				.map_err(|_| CallError::StateCorrupt)?;
-		}
 		let options = TransactOptions { tracing: analytics.transaction_tracing, vm_tracing: analytics.vm_tracing, check_nonce: false };
-		let mut ret = Executive::new(&mut state, &env_info, &*self.engine, &self.factories.vm).transact(t, options)?;
+		let mut ret = Executive::new(&mut state, &env_info, &*self.engine, &self.factories.vm)
+			.transact_virtual(t, options)?;
 
 		// TODO gav move this into Executive.
 		if let Some(original) = original_state {
@@ -1023,7 +1055,6 @@ impl BlockChainClient for Client {
 		// that's just a copy of the state.
 		let original_state = self.state_at(block).ok_or(CallError::StatePruned)?;
 		let sender = t.sender();
-		let balance = original_state.balance(&sender).map_err(ExecutionError::from)?;
 		let options = TransactOptions { tracing: true, vm_tracing: false, check_nonce: false };
 
 		let cond = |gas| {
@@ -1032,15 +1063,8 @@ impl BlockChainClient for Client {
 			let tx = tx.fake_sign(sender);
 
 			let mut state = original_state.clone();
-			let needed_balance = tx.value + tx.gas * tx.gas_price;
-			if balance < needed_balance {
-				// give the sender a sufficient balance
-				state.add_balance(&sender, &(needed_balance - balance), CleanupMode::NoEmpty)
-					.map_err(ExecutionError::from)?;
-			}
-
 			Ok(Executive::new(&mut state, &env_info, &*self.engine, &self.factories.vm)
-				.transact(&tx, options.clone())
+				.transact_virtual(&tx, options.clone())
 				.map(|r| r.exception.is_none())
 				.unwrap_or(false))
 		};

ethcore/src/client/mod.rs

@@ -16,6 +16,7 @@
 
 //! Blockchain database client.
 
+mod ancient_import;
 mod config;
 mod error;
 mod test_client;